private Path executeWindupAgainstAppUntilRule( final String inputDir, final GraphContext grCtx, final Class<MavenizeRuleProvider> ruleToRunUpTo) throws IOException, IllegalAccessException, InstantiationException { Assume.assumeTrue("Exists: " + inputDir, new File(inputDir).exists()); final Path outputPath = Paths.get(FileUtils.getTempDirectory().toString(), "Windup-Mavenization-output"); FileUtils.deleteDirectory(outputPath.toFile()); Files.createDirectories(outputPath); grCtx.getGraph().getBaseGraph().commit(); // Configure Windup core final WindupConfiguration processorConfig = new WindupConfiguration(); processorConfig.setRuleProviderFilter(new RuleProviderWithDependenciesPredicate(ruleToRunUpTo)); processorConfig.setGraphContext(grCtx); processorConfig.addInputPath(Paths.get(inputDir)); processorConfig.setOutputDirectory(outputPath); processorConfig.setOptionValue(ScanPackagesOption.NAME, Collections.singletonList("")); processorConfig.setOptionValue(SourceModeOption.NAME, false); processorConfig.setOptionValue(MavenizeOption.NAME, true); processor.execute(processorConfig); return outputPath; }
/** Take a screenshot of the current browser and store it in the output directory. */ public File takeScreenshot(final String prefix) { File screenshot = null; if (driverCanTakeSnapshots()) { OutputStream stream = null; try { byte[] screenshotData = ((TakesScreenshot) driver).getScreenshotAs(OutputType.BYTES); if (screenshotData != null) { File temporaryFolder = FileUtils.getTempDirectory(); String snapshotName = getTemporarySnapshotName(); screenshot = new File(temporaryFolder, snapshotName); stream = new FileOutputStream(screenshot); stream.write(screenshotData); } if ((screenshot != null) && (screenshot.exists())) { return saveScreenshoot(prefix, screenshot); } else if (!isAMock(driver)) { getLogger().warn("Failed to write screenshot (possibly an out of memory error)"); } } catch (Throwable e) { getLogger() .warn( "Failed to write screenshot (possibly an out of memory error): " + e.getMessage()); } finally { if (stream != null) try { stream.close(); } catch (IOException e) { } // Ignore any error on close } } return screenshot; }
private MultiFilesResult buildMockMultiFilesCallResult() throws IOException { final MultiFilesResult multiFilesResult = mock(MultiFilesResult.class); when(multiFilesResult.getApplicationName()).thenReturn("test_app_name"); when(multiFilesResult.getPayload()).thenReturn(new File[0]); when(multiFilesResult.getTemporaryDirectory()).thenReturn(FileUtils.getTempDirectory()); return multiFilesResult; }
/** * Downloads the given file specified via url to the given canonicalDestination. * * @param urlSource String * @param urlDestination String * @throws Exception */ @Override public void downloadFile(String urlSource, String urlDestination) throws Exception { // sanity check if (urlSource == null || urlSource.length() == 0 || urlDestination == null || urlDestination.length() == 0) { throw new IllegalArgumentException( "downloadFile(): urlSource or urlDestination argument is null..."); } // URLs for given parameters URL source = new URL(urlSource); URL destination = new URL(urlDestination); // we have a compressed file if (GzipUtils.isCompressedFilename(urlSource)) { // downlod to temp destination File tempDestinationFile = org.apache.commons.io.FileUtils.getFile( org.apache.commons.io.FileUtils.getTempDirectory(), new File(source.getFile()).getName()); if (LOG.isInfoEnabled()) { LOG.info("downloadFile(), " + urlSource + ", this may take a while..."); } org.apache.commons.io.FileUtils.copyURLToFile(source, tempDestinationFile); if (LOG.isInfoEnabled()) { LOG.info("downloadFile(), gunzip: we have compressed file, decompressing..."); } // decompress the file gunzip(tempDestinationFile.getCanonicalPath()); if (LOG.isInfoEnabled()) { LOG.info("downloadFile(), gunzip complete..."); } // move temp/decompressed file to final destination File destinationFile = new File(destination.getFile()); if (destinationFile.exists()) { org.apache.commons.io.FileUtils.forceDelete(destinationFile); } org.apache.commons.io.FileUtils.moveFile( org.apache.commons.io.FileUtils.getFile( GzipUtils.getUncompressedFilename(tempDestinationFile.getCanonicalPath())), destinationFile); // lets cleanup after ourselves - remove compressed file tempDestinationFile.delete(); } // uncompressed file, download directry to urlDestination else { if (LOG.isInfoEnabled()) { LOG.info("downloadFile(), " + urlSource + ", this may take a while..."); } org.apache.commons.io.FileUtils.copyURLToFile( source, org.apache.commons.io.FileUtils.getFile(destination.getFile())); } }
public AndroidApp createSelendroidServer(AndroidApp aut) throws IOException, ShellCommandException, AndroidSdkException { log.info("create SelendroidServer for apk: " + aut.getAbsolutePath()); init(aut); cleanUpPrebuildServer(); File selendroidServer = createAndAddCustomizedAndroidManifestToSelendroidServer(); File outputFile = new File( FileUtils.getTempDirectory(), String.format( "selendroid-server-%s-%s.apk", applicationUnderTest.getBasePackage(), getJarVersionNumber())); return signTestServer(selendroidServer, outputFile); }
/** * Creates a staging file for mutation data (and meta file) with contents from the given * DataMatrix. This is called when the mutation file needs to be run through the Oncotator and * Mutation Assessor Tools. * * @param portalMetadata PortalMetadata * @param cancerStudy CancerStudyMetadata * @param datatypeMetadata DatatypeMetadata * @param dataMatrix DataMatrix * @throws Exception */ @Override public void writeMutationStagingFile( PortalMetadata portalMetadata, CancerStudyMetadata cancerStudyMetadata, DatatypeMetadata datatypeMetadata, DataMatrix dataMatrix) throws Exception { // we only have data matrix at this point, we need to create a temp with its contents File oncotatorInputFile = org.apache.commons.io.FileUtils.getFile( org.apache.commons.io.FileUtils.getTempDirectory(), "oncotatorInputFile"); FileOutputStream out = org.apache.commons.io.FileUtils.openOutputStream(oncotatorInputFile); dataMatrix.write(out); IOUtils.closeQuietly(out); // output should be the path/name of staging file String stagingFilename = datatypeMetadata.getStagingFilename(); stagingFilename = stagingFilename.replaceAll( DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString()); File stagingFile = org.apache.commons.io.FileUtils.getFile( portalMetadata.getStagingDirectory(), cancerStudyMetadata.getStudyPath(), stagingFilename); // call oncotateAF oncotateMAF( FileUtils.FILE_URL_PREFIX + oncotatorInputFile.getCanonicalPath(), FileUtils.FILE_URL_PREFIX + stagingFile.getCanonicalPath()); // clean up if (oncotatorInputFile.exists()) { org.apache.commons.io.FileUtils.forceDelete(oncotatorInputFile); } // meta file if (datatypeMetadata.requiresMetafile()) { if (LOG.isInfoEnabled()) { LOG.info( "writingMutationStagingFile(), creating metadata file for staging file: " + stagingFile); } writeMetadataFile(portalMetadata, cancerStudyMetadata, datatypeMetadata, dataMatrix); } }
@Override public boolean execute() throws Exception { if (_controllerHost == null) { _controllerHost = NetUtil.getHostAddress(); } // Create a temp working directory. File tmpDir = File.createTempFile(SEGMENT_UPLOADER, null, FileUtils.getTempDirectory()); FileUtils.deleteQuietly(tmpDir); tmpDir.mkdir(); try { LOGGER.info("Executing command: " + toString()); File dir = new File(_segmentDir); File[] files = dir.listFiles(); for (File file : files) { File tgzFile = file; if (file.isDirectory()) { LOGGER.info("Compressing segment {}", file.getName()); String srcDir = file.getAbsolutePath(); String tgzFileName = TarGzCompressionUtils.createTarGzOfDirectory( srcDir, tmpDir.getAbsolutePath() + File.separator + file.getName() + TAR_GZIP); tgzFile = new File(tgzFileName); } LOGGER.info("Uploading segment {}", tgzFile.getName()); FileUploadUtils.sendSegmentFile( _controllerHost, _controllerPort, tgzFile.getName(), new FileInputStream(tgzFile), tgzFile.length()); } } catch (Exception e) { LOGGER.error("Exception caught while uploading segment {}", _segmentDir, e); } finally { // Delete the temporary working directory. FileUtils.deleteQuietly(tmpDir); } return true; }
@Test public void createByUrl() throws IOException { InputStream in = getClass().getResourceAsStream("/testdata.csv"); File csvFile = new File(FileUtils.getTempDirectory(), "testdata.csv"); FileCopyUtils.copy(in, new FileOutputStream(csvFile)); String url = "csv://" + csvFile.getAbsolutePath(); EntitySource entitySource = new CsvEntitySource(url, null); try { assertEquals(entitySource.getUrl(), url); Iterator<String> it = entitySource.getEntityNames().iterator(); assertNotNull(it); assertTrue(it.hasNext()); assertEquals(it.next(), "testdata"); assertFalse(it.hasNext()); Repository<? extends Entity> repo = entitySource.getRepositoryByEntityName("testdata"); assertNotNull(repo); Iterator<AttributeMetaData> itMeta = repo.getAttributes().iterator(); assertNotNull(itMeta); assertTrue(itMeta.hasNext()); AttributeMetaData col1 = itMeta.next(); assertNotNull(col1); assertEquals(col1.getName(), "col1"); AttributeMetaData col2 = itMeta.next(); assertNotNull(col2); assertEquals(col2.getName(), "col2"); assertFalse(itMeta.hasNext()); Iterator<? extends Entity> itEntity = repo.iterator(); assertNotNull(itEntity); assertTrue(itEntity.hasNext()); Entity entity = itEntity.next(); assertNotNull(entity); assertEquals(entity.get("col1"), "val1"); assertEquals(entity.get("col2"), "val2"); } finally { entitySource.close(); } }
@Test public void testNestedCondition() throws IOException { try (GraphContext context = factory.create()) { ProjectModel pm = context.getFramed().addVertex(null, ProjectModel.class); pm.setName("Main Project"); FileModel inputPath = context.getFramed().addVertex(null, FileModel.class); inputPath.setFilePath("src/test/resources/"); Path outputPath = Paths.get( FileUtils.getTempDirectory().toString(), "windup_" + UUID.randomUUID().toString()); FileUtils.deleteDirectory(outputPath.toFile()); Files.createDirectories(outputPath); inputPath.setProjectModel(pm); pm.setRootFileModel(inputPath); WindupConfiguration windupConfiguration = new WindupConfiguration() .setRuleProviderFilter( new NotPredicate( new RuleProviderPhasePredicate( MigrationRulesPhase.class, ReportGenerationPhase.class))) .setGraphContext(context); windupConfiguration.setInputPath(Paths.get(inputPath.getFilePath())); windupConfiguration.setOutputDirectory(outputPath); processor.execute(windupConfiguration); GraphService<ClassificationModel> classificationService = new GraphService<>(context, ClassificationModel.class); Assert.assertEquals(1, provider.getXmlFileMatches().size()); List<ClassificationModel> classifications = Iterators.asList(classificationService.findAll()); for (ClassificationModel model : classifications) { String classification = model.getClassification(); String classificationString = classification.toString(); Assert.assertEquals("Spring File", classificationString); } Assert.assertEquals(1, classifications.size()); Iterator<FileModel> iterator = classifications.get(0).getFileModels().iterator(); Assert.assertNotNull(iterator.next()); Assert.assertNotNull(iterator.next()); Assert.assertFalse(iterator.hasNext()); } }
public static void main(String[] args) throws Exception { File tmpDir = new File(FileUtils.getTempDirectory(), "macc-" + UUID.randomUUID().toString()); try { MiniAccumuloCluster la = new MiniAccumuloCluster(tmpDir, "pass1234", new HashMap<String, String>()); la.start(); System.out.println( "\n ---- Running Accumulo App against accumulo-" + la.getAccumuloVersion() + "\n"); run(la.getInstanceName(), la.getZookeepers(), new PasswordToken("pass1234"), args); System.out.println("\n ---- Ran Accumulo App\n"); la.stop(); } finally { FileUtils.deleteQuietly(tmpDir); } }
/** * Runs all MAFs for the given dataaSourcesMetadata through the Oncotator and OMA tools. * * @param dataSourcesMetadata DataSourcesMetadata * @throws Exception */ @Override public void oncotateAllMAFs(DataSourcesMetadata dataSourcesMetadata) throws Exception { // iterate over datasource download directory and process all MAFs String[] extensions = new String[] {DatatypeMetadata.MAF_FILE_EXT}; for (File maf : listFiles(new File(dataSourcesMetadata.getDownloadDirectory()), extensions, true)) { // create temp for given maf File oncotatorInputFile = org.apache.commons.io.FileUtils.getFile( org.apache.commons.io.FileUtils.getTempDirectory(), "oncotatorInputFile"); org.apache.commons.io.FileUtils.copyFile(maf, oncotatorInputFile); // input is tmp file we just created, we want output to go into the original maf oncotateMAF( FileUtils.FILE_URL_PREFIX + oncotatorInputFile.getCanonicalPath(), FileUtils.FILE_URL_PREFIX + maf.getCanonicalPath()); // clean up org.apache.commons.io.FileUtils.forceDelete(oncotatorInputFile); } }
@Test public void testRunWindupSourceMode() throws Exception { Path userPath = FileUtils.getTempDirectory() .toPath() .resolve("Windup") .resolve("windupuserscriptsdir_" + RandomStringUtils.randomAlphanumeric(6)); try { Files.createDirectories(userPath); try (InputStream is = getClass().getResourceAsStream(EXAMPLE_USERSCRIPT_INPUT); OutputStream os = new FileOutputStream(userPath.resolve(EXAMPLE_USERSCRIPT_OUTPUT).toFile())) { IOUtils.copy(is, os); } try (InputStream is = getClass().getResourceAsStream("/exampleconversion.xsl"); OutputStream os = new FileOutputStream(userPath.resolve(XSLT_OUTPUT_NAME).toFile())) { IOUtils.copy(is, os); } try (GraphContext context = createGraphContext()) { // The test-files folder in the project root dir. List<String> includeList = Collections.emptyList(); List<String> excludeList = Collections.emptyList(); super.runTest( context, "../test-files/src_example", userPath.toFile(), true, includeList, excludeList); validateWebXmlReferences(context); validatePropertiesModels(context); validateReports(context); validateCompatibleReport(context); } } finally { FileUtils.deleteDirectory(userPath.toFile()); } }
@Test public void testSimpleSchema() throws Exception { CompiledDatum result = DatumCompiler.generate(MyDatumTemplate.class); File baseDir = FileUtils.getTempDirectory(); FileUtils.deleteDirectory(baseDir); File srcDir = new File(baseDir, result.getPackageName().replaceAll("\\.", "/")); assertTrue(srcDir.mkdirs()); File codeFile = new File(srcDir, result.getClassName() + ".java"); OutputStream os = new FileOutputStream(codeFile); IOUtils.write(result.getClassCode(), os, "UTF-8"); os.close(); // Compile with Janino, give it a try. We have Janino since // it's a cascading dependency, but probably want to add a test // dependency on it. ClassLoader cl = new JavaSourceClassLoader( this.getClass().getClassLoader(), // parentClassLoader new File[] {baseDir}, // optionalSourcePath (String) null // optionalCharacterEncoding ); // WARNING - we have to use xxxDatumTemplate as the base name, so that the code returned // by the compiler is for type xxxDatum. Otherwise when we try to load the class here, // we'll likely get the base (template) class, which will mask our generated class. Class clazz = cl.loadClass(result.getPackageName() + "." + result.getClassName()); assertEquals("MyDatum", clazz.getSimpleName()); // Verify that we have a constructor which takes all of the fields. // private String _name; // private int ageAndRisk; // private Date _date; // private Tuple _aliases; Constructor c = clazz.getConstructor(String.class, int.class, Date.class, Tuple.class); Object datum = c.newInstance("robert", 25, new Date(), new Tuple("bob", "rob")); }
/** * Reads the script from the resource, processes the variables and writes it to the file system so * it can be accessed by PhantomJS. * * @throws IOException if reading the template or writing the file failed */ private void prepareScript() throws IOException { InputStream is = null; InputStream fis = null; OutputStream os = null; try { // Create the temporary directory for everything PhantomJS phantomTmpDir = new File(FileUtils.getTempDirectory(), "phantomjs"); if (!phantomTmpDir.isDirectory() && !phantomTmpDir.mkdirs()) { logger.error("Unable to create temp directory for PhantomJS at {}", phantomTmpDir); throw new IOException("Unable to create temp directory for PhantomJS at " + phantomTmpDir); } // Create the script is = PhantomJsPagePreviewGenerator.class.getResourceAsStream(SCRIPT_FILE); scriptTemplate = IOUtils.toString(is); scriptFile = new File(phantomTmpDir, "pagepreview.js"); // Process templates Map<String, String> properties = new HashMap<String, String>(); properties.put(PARAM_PREPARE_SCRIPT, "return true;"); String script = ConfigurationUtils.processTemplate(scriptTemplate, properties); // Write the processed script to disk fis = IOUtils.toInputStream(script); os = new FileOutputStream(scriptFile); IOUtils.copy(fis, os); } catch (IOException e) { logger.error("Error reading phantomjs script template from " + SCRIPT_FILE, e); FileUtils.deleteQuietly(scriptFile); throw e; } finally { IOUtils.closeQuietly(is); IOUtils.closeQuietly(fis); IOUtils.closeQuietly(os); } }
private File unpack(File dest) throws IOException { ZipFile zfile = new ZipFile(this.getZip()); Enumeration<? extends ZipEntry> entries = zfile.entries(); File f, basedir = null; File tmp = FileUtils.getTempDirectory(); while (entries.hasMoreElements()) { ZipEntry entry = entries.nextElement(); f = new File(tmp, entry.getName()); if (f.exists()) FileUtils.deleteQuietly(f); if (entry.isDirectory()) FileUtils.forceMkdir(f); else FileUtils.copyInputStreamToFile(zfile.getInputStream(entry), f); if (entry.isDirectory() && (basedir == null || basedir.getName().startsWith(f.getName()))) basedir = f; } zfile.close(); if (dest.exists()) FileUtils.deleteDirectory(dest); FileUtils.moveDirectory(basedir, dest); return dest; }
/** Oct 14, 2014 */ public class QueriesSentinelTest { private static final Logger LOGGER = LoggerFactory.getLogger(QueriesSentinelTest.class); private static ReduceService REDUCE_SERVICE = new DefaultReduceService(); private static final PQLCompiler REQUEST_COMPILER = new PQLCompiler(new HashMap<String, String[]>()); private final String AVRO_DATA = "data/test_data-mv.avro"; private static File INDEX_DIR = new File(FileUtils.getTempDirectory() + File.separator + "QueriesSentinelTest"); private static AvroQueryGenerator AVRO_QUERY_GENERATOR; private static QueryExecutor QUERY_EXECUTOR; private static TestingServerPropertiesBuilder CONFIG_BUILDER; private String segmentName; @BeforeClass public void setup() throws Exception { CONFIG_BUILDER = new TestingServerPropertiesBuilder("testTable"); setupSegmentFor("testTable"); setUpTestQueries("testTable"); final PropertiesConfiguration serverConf = CONFIG_BUILDER.build(); serverConf.setDelimiterParsingDisabled(false); final FileBasedInstanceDataManager instanceDataManager = FileBasedInstanceDataManager.getInstanceDataManager(); instanceDataManager.init( new FileBasedInstanceDataManagerConfig(serverConf.subset("pinot.server.instance"))); instanceDataManager.start(); System.out.println( "************************** : " + new File(INDEX_DIR, "segment").getAbsolutePath()); File segmentFile = new File(INDEX_DIR, "segment").listFiles()[0]; segmentName = segmentFile.getName(); final IndexSegment indexSegment = ColumnarSegmentLoader.load(segmentFile, ReadMode.heap); instanceDataManager.getTableDataManager("testTable"); instanceDataManager.getTableDataManager("testTable").addSegment(indexSegment); QUERY_EXECUTOR = new ServerQueryExecutorV1Impl(false); QUERY_EXECUTOR.init( serverConf.subset("pinot.server.query.executor"), instanceDataManager, new ServerMetrics(new MetricsRegistry())); } @AfterClass public void tearDown() { FileUtils.deleteQuietly(INDEX_DIR); } private void runApproximationQueries( List<? extends AvroQueryGenerator.TestAggreationQuery> queries, double precision) throws Exception { boolean isAccurate = true; Object accurateValue = null; int counter = 0; final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); for (final AvroQueryGenerator.TestAggreationQuery query : queries) { LOGGER.info("**************************"); LOGGER.info("running " + counter + " : " + query.getPql()); final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query.getPql())); InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); instanceRequest.getSearchSegments().add(segmentName); final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest); instanceResponseMap.clear(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); final BrokerResponse brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0)); // compute value Object val; if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) { val = Double.parseDouble(brokerResponse.getAggregationResults().get(0).getString("value")); } else { val = brokerResponse.getAggregationResults().get(0).getJSONArray("groupByResult"); } if (isAccurate) { // store accurate value accurateValue = val; isAccurate = false; } else { // compare value with accurate value // it's estimation so we need to test its result within error bound if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) { TestUtils.assertApproximation((Double) val, (Double) accurateValue, precision); } else { TestUtils.assertJSONArrayApproximation( (JSONArray) val, (JSONArray) accurateValue, precision); } isAccurate = true; } } } /** * Console output of the last statement may not appear, maybe a result of intellij idea test * console redirection. To avoid this, always add assert clauses, and do not rely on the console * output. * * @throws Exception */ @Test public void testDistinctCountHLLNoGroupBy() throws Exception { final List<TestSimpleAggreationQuery> aggCalls = new ArrayList<TestSimpleAggreationQuery>(); // distinct count(*) not works for (int i = 1; i <= 5; i++) { aggCalls.add( new TestSimpleAggreationQuery( "select distinctcount(column" + i + ") from testTable limit 0", 0.0)); aggCalls.add( new TestSimpleAggreationQuery( "select distinctcounthll(column" + i + ") from testTable limit 0", 0.0)); } runApproximationQueries(aggCalls, 0.1); } @Test public void testDistinctCountHLLGroupBy() throws Exception { final List<TestGroupByAggreationQuery> groupByCalls = new ArrayList<TestGroupByAggreationQuery>(); for (int i = 1; i <= 5; i++) { if (i == 2) { continue; } groupByCalls.add( new TestGroupByAggreationQuery( "select distinctcount(column2) from testTable group by column" + i + " limit 0", null)); groupByCalls.add( new TestGroupByAggreationQuery( "select distinctcounthll(column2) from testTable group by column" + i + " limit 0", null)); } runApproximationQueries(groupByCalls, 0.1); } @Test public void testPercentileNoGroupBy() throws Exception { final List<TestSimpleAggreationQuery> aggCalls = new ArrayList<TestSimpleAggreationQuery>(); // 5 single-value columns -- column 3 is String type for (int i = 1; i <= 2; i++) { aggCalls.add( new TestSimpleAggreationQuery( "select percentile50(column" + i + ") from testTable limit 0", 0.0)); aggCalls.add( new TestSimpleAggreationQuery( "select percentileest50(column" + i + ") from testTable limit 0", 0.0)); } runApproximationQueries(aggCalls, 0.1); } @Test public void testPercentileGroupBy() throws Exception { final List<TestGroupByAggreationQuery> groupByCalls = new ArrayList<TestGroupByAggreationQuery>(); final int top = 1000; for (int i = 2; i <= 2; i++) { if (i == 2) { // continue; } groupByCalls.add( new TestGroupByAggreationQuery( "select percentile50(column1) from testTable group by column" + i + " top " + top + " limit 0", null)); groupByCalls.add( new TestGroupByAggreationQuery( "select percentileest50(column1) from testTable group by column" + i + " top " + top + " limit 0", null)); } runApproximationQueries(groupByCalls, 0.1); } @Test public void testAggregation() throws Exception { int counter = 0; final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); final List<TestSimpleAggreationQuery> aggCalls = AVRO_QUERY_GENERATOR.giveMeNSimpleAggregationQueries(10000); for (final TestSimpleAggreationQuery aggCall : aggCalls) { LOGGER.info("running " + counter + " : " + aggCall.pql); final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(aggCall.pql)); InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); instanceRequest.getSearchSegments().add(segmentName); final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest); instanceResponseMap.clear(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); final BrokerResponse brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0)); LOGGER.info("Result from avro is : " + aggCall.result); try { Assert.assertEquals( Double.parseDouble(brokerResponse.getAggregationResults().get(0).getString("value")), aggCall.result); } catch (AssertionError e) { System.out.println(aggCall.pql); System.out.println( "from broker : " + Double.parseDouble( brokerResponse.getAggregationResults().get(0).getString("value"))); System.out.println("from precomp : " + aggCall.result); throw new AssertionError(e); } } } @Test public void testAggregationGroupBy() throws Exception { final List<TestGroupByAggreationQuery> groupByCalls = AVRO_QUERY_GENERATOR.giveMeNGroupByAggregationQueries(10000); int counter = 0; final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); for (final TestGroupByAggreationQuery groupBy : groupByCalls) { LOGGER.info("running " + counter + " : " + groupBy.pql); final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(groupBy.pql)); InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); instanceRequest.getSearchSegments().add(segmentName); final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest); instanceResponseMap.clear(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); final BrokerResponse brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0)); LOGGER.info("Result from avro is : " + groupBy.groupResults); try { assertGroupByResults( brokerResponse.getAggregationResults().get(0).getJSONArray("groupByResult"), groupBy.groupResults); } catch (AssertionError e) { System.out.println(groupBy.pql); System.out.println( "from broker : " + brokerResponse .getAggregationResults() .get(0) .getJSONArray("groupByResult") .toString()); System.out.println("from precomp : " + groupBy.groupResults); throw new AssertionError(e); } } } private void assertGroupByResults(JSONArray jsonArray, Map<Object, Double> groupResultsFromAvro) throws JSONException { final Map<String, Double> groupResultsFromQuery = new HashMap<String, Double>(); if (groupResultsFromAvro.size() > 10) { Assert.assertEquals(jsonArray.length(), 10); } else { Assert.assertTrue(jsonArray.length() >= groupResultsFromAvro.size()); } for (int i = 0; i < jsonArray.length(); ++i) { groupResultsFromQuery.put( jsonArray.getJSONObject(i).getJSONArray("group").getString(0), jsonArray.getJSONObject(i).getDouble("value")); } for (final Object key : groupResultsFromAvro.keySet()) { String keyString; if (key == null) { keyString = "null"; } else { keyString = key.toString(); } if (!groupResultsFromQuery.containsKey(keyString)) { continue; } final double actual = groupResultsFromQuery.get(keyString); // System.out.println("Result from query - group:" + keyString + ", value:" + actual); final double expected = groupResultsFromAvro.get(key); // System.out.println("Result from avro - group:" + keyString + ", value:" + expected); try { Assert.assertEquals(actual, expected); } catch (AssertionError e) { throw new AssertionError(e); } } } private void setUpTestQueries(String table) throws FileNotFoundException, IOException { final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA)); System.out.println(filePath); final List<String> dims = new ArrayList<String>(); dims.add("column1"); dims.add("column2"); dims.add("column3"); dims.add("column4"); dims.add("column5"); dims.add("column6"); dims.add("column7"); dims.add("column8"); dims.add("column9"); dims.add("column10"); dims.add("weeksSinceEpochSunday"); dims.add("daysSinceEpoch"); dims.add("count"); final List<String> mets = new ArrayList<String>(); mets.add("count"); final String time = "minutesSinceEpoch"; AVRO_QUERY_GENERATOR = new AvroQueryGenerator(new File(filePath), dims, mets, time, table); AVRO_QUERY_GENERATOR.init(); AVRO_QUERY_GENERATOR.generateSimpleAggregationOnSingleColumnFilters(); } private void setupSegmentFor(String table) throws Exception { final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA)); if (INDEX_DIR.exists()) { FileUtils.deleteQuietly(INDEX_DIR); } INDEX_DIR.mkdir(); final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns( new File(filePath), new File(INDEX_DIR, "segment"), "daysSinceEpoch", TimeUnit.DAYS, table); final SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl(); driver.init(config); driver.build(); System.out.println("built at : " + INDEX_DIR.getAbsolutePath()); } @Test public void testSingleQuery() throws RecognitionException, Exception { String query; query = "select count(*) from testTable where column5='kCMyNVGCASKYDdQbftOPaqVMWc'"; // query= "select sum('count') from testTable where column1='660156454'"; LOGGER.info("running : " + query); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query)); InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); instanceRequest.getSearchSegments().add(segmentName); final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest); instanceResponseMap.clear(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); final BrokerResponse brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0)); } @Test public void testMatchAllQuery() throws RecognitionException, Exception { String query = "select count(*),sum(count) from testTable "; LOGGER.info("running : " + query); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query)); InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); instanceRequest.getSearchSegments().add(segmentName); final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest); instanceResponseMap.clear(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); final BrokerResponse brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.info("BrokerResponse is " + brokerResponse); LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0)); LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(1)); Assert.assertEquals(brokerResponse.getAggregationResults().get(0).getInt("value"), 100000); Assert.assertEquals( brokerResponse.getAggregationResults().get(1).getDouble("value"), 8.90662862E13); Assert.assertEquals(brokerResponse.getNumDocsScanned(), 100000); } @Test public void testRangeQuery() throws RecognitionException, Exception { String query = "select count(*) from testTable where column1 in ('999983251', '510705831', '1000720716', '1001058817', '1001099410')"; LOGGER.info("running : " + query); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query)); InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); instanceRequest.getSearchSegments().add(segmentName); final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest); instanceResponseMap.clear(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); final BrokerResponse brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0)); Assert.assertEquals(brokerResponse.getAggregationResults().get(0).getInt("value"), 14); Assert.assertEquals(brokerResponse.getNumDocsScanned(), 14); } @Test public void testTrace() throws RecognitionException, Exception { String query = "select count(*) from testTable where column1='186154188'"; LOGGER.info("running : " + query); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query)); brokerRequest.setEnableTrace(true); // InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest); instanceRequest.setEnableTrace(true); // TODO: add trace settings consistency instanceRequest.setSearchSegments(new ArrayList<String>()); instanceRequest.getSearchSegments().add(segmentName); final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest); instanceResponseMap.clear(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); final BrokerResponse brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0)); LOGGER.info("TraceInfo is " + brokerResponse.getTraceInfo()); // } }
public class AggregationGroupByWithDictionaryAndTrieTreeOperatorTest { protected static Logger LOGGER = LoggerFactory.getLogger(AggregationGroupByWithDictionaryAndTrieTreeOperatorTest.class); private final String AVRO_DATA = "data/test_sample_data.avro"; private static File INDEX_DIR = new File( FileUtils.getTempDirectory() + File.separator + "TestAggregationGroupByWithDictionaryAndTrieTreeOperator"); private static File INDEXES_DIR = new File( FileUtils.getTempDirectory() + File.separator + "TestAggregationGroupByWithDictionaryAndTrieTreeOperatorList"); public static IndexSegment _indexSegment; private static List<SegmentDataManager> _indexSegmentList; public static AggregationInfo _paramsInfo; public static List<AggregationInfo> _aggregationInfos; public static int _numAggregations = 6; public Map<String, ColumnMetadata> _medataMap; public static GroupBy _groupBy; @BeforeClass public void setup() throws Exception { setupSegment(); setupQuery(); _indexSegmentList = new ArrayList<SegmentDataManager>(); } @AfterClass public void tearDown() { if (INDEX_DIR.exists()) { FileUtils.deleteQuietly(INDEX_DIR); } if (INDEXES_DIR.exists()) { FileUtils.deleteQuietly(INDEXES_DIR); } } private void setupSegment() throws Exception { final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA)); if (INDEX_DIR.exists()) { FileUtils.deleteQuietly(INDEX_DIR); } final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns( new File(filePath), INDEX_DIR, "time_day", TimeUnit.DAYS, "test"); final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null); driver.init(config); driver.build(); LOGGER.debug("built at : {}", INDEX_DIR.getAbsolutePath()); final File indexSegmentDir = new File(INDEX_DIR, driver.getSegmentName()); _indexSegment = ColumnarSegmentLoader.load(indexSegmentDir, ReadMode.heap); _medataMap = ((SegmentMetadataImpl) ((IndexSegmentImpl) _indexSegment).getSegmentMetadata()) .getColumnMetadataMap(); } public void setupQuery() { _aggregationInfos = getAggregationsInfo(); final List<String> groupbyColumns = new ArrayList<String>(); groupbyColumns.add("column11"); _groupBy = new GroupBy(); _groupBy.setColumns(groupbyColumns); _groupBy.setTopN(10); } private void setupSegmentList(int numberOfSegments) throws Exception { final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA)); _indexSegmentList.clear(); if (INDEXES_DIR.exists()) { FileUtils.deleteQuietly(INDEXES_DIR); } INDEXES_DIR.mkdir(); for (int i = 0; i < numberOfSegments; ++i) { final File segmentDir = new File(INDEXES_DIR, "segment_" + i); final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns( new File(filePath), segmentDir, "time_day", TimeUnit.DAYS, "test"); config.setSegmentNamePostfix(String.valueOf(i)); final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null); driver.init(config); driver.build(); LOGGER.debug("built at : {}", segmentDir.getAbsolutePath()); final File indexSegmentDir = new File(segmentDir, driver.getSegmentName()); _indexSegmentList.add( new OfflineSegmentDataManager( ColumnarSegmentLoader.load(indexSegmentDir, ReadMode.heap))); } } @Test public void testAggregationGroupBys() { final MAggregationGroupByOperator aggregationGroupByOperator = getmAggregationGroupByOperator(); LOGGER.debug("running query: "); final IntermediateResultsBlock block = (IntermediateResultsBlock) aggregationGroupByOperator.nextBlock(); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Result: {}", block.getAggregationGroupByOperatorResult().get(i)); } } private MAggregationGroupByOperator getmAggregationGroupByOperator() { final List<AggregationFunctionGroupByOperator> aggregationFunctionGroupByOperatorList = new ArrayList<AggregationFunctionGroupByOperator>(); Operator filterOperator = new MatchEntireSegmentOperator(_indexSegment.getSegmentMetadata().getTotalRawDocs()); final BReusableFilteredDocIdSetOperator docIdSetOperator = new BReusableFilteredDocIdSetOperator( filterOperator, _indexSegment.getSegmentMetadata().getTotalRawDocs(), 5000); final Map<String, DataSource> dataSourceMap = getDataSourceMap(); final MProjectionOperator projectionOperator = new MProjectionOperator(dataSourceMap, docIdSetOperator); for (int i = 0; i < _numAggregations; ++i) { final MAggregationFunctionGroupByWithDictionaryAndTrieTreeOperator aggregationFunctionGroupByOperator = new MAggregationFunctionGroupByWithDictionaryAndTrieTreeOperator( _aggregationInfos.get(i), _groupBy, new UReplicatedProjectionOperator(projectionOperator), true); aggregationFunctionGroupByOperatorList.add(aggregationFunctionGroupByOperator); } return new MAggregationGroupByOperator( _indexSegment, _aggregationInfos, _groupBy, projectionOperator, aggregationFunctionGroupByOperatorList); } @Test public void testAggregationGroupBysWithCombine() { final MAggregationGroupByOperator aggregationGroupByOperator = getmAggregationGroupByOperator(); final IntermediateResultsBlock block = (IntermediateResultsBlock) aggregationGroupByOperator.nextBlock(); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Result1: {}", block.getAggregationGroupByOperatorResult().get(i)); } //////////////////////////////////////////////////////////////////////////////// final MAggregationGroupByOperator aggregationGroupByOperator1 = getmAggregationGroupByOperator(); final IntermediateResultsBlock block1 = (IntermediateResultsBlock) aggregationGroupByOperator1.nextBlock(); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Result 2: {}", block1.getAggregationGroupByOperatorResult().get(i)); } CombineService.mergeTwoBlocks(getAggregationGroupByNoFilterBrokerRequest(), block, block1); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Combined Result: {}", block.getAggregationGroupByOperatorResult().get(i)); } } @Test public void testAggregationGroupBysWithDataTableEncodeAndDecode() throws Exception { final MAggregationGroupByOperator aggregationGroupByOperator = getmAggregationGroupByOperator(); final IntermediateResultsBlock block = (IntermediateResultsBlock) aggregationGroupByOperator.nextBlock(); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Result 1: {}", block.getAggregationGroupByOperatorResult().get(i)); } //////////////////////////////////////////////////////////////////////////////// final MAggregationGroupByOperator aggregationGroupByOperator1 = getmAggregationGroupByOperator(); final IntermediateResultsBlock block1 = (IntermediateResultsBlock) aggregationGroupByOperator1.nextBlock(); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Result 2: {}", block1.getAggregationGroupByOperatorResult().get(i)); } CombineService.mergeTwoBlocks(getAggregationGroupByNoFilterBrokerRequest(), block, block1); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Combined Result: {}", block.getAggregationGroupByOperatorResult().get(i)); } final DataTable dataTable = block.getAggregationGroupByResultDataTable(); final List<Map<String, Serializable>> results = AggregationGroupByOperatorService.transformDataTableToGroupByResult(dataTable); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Decoded Result: {}", results.get(i)); } } @Test public void testInnerSegmentPlanMakerForAggregationGroupByOperatorNoFilter() throws Exception { final BrokerRequest brokerRequest = getAggregationGroupByNoFilterBrokerRequest(); final PlanMaker instancePlanMaker = new InstancePlanMakerImplV1(); final PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); final MAggregationGroupByOperator operator = (MAggregationGroupByOperator) rootPlanNode.run(); final IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) operator.nextBlock(); LOGGER.debug("RunningTime : {}", resultBlock.getTimeUsedMs()); LOGGER.debug("NumDocsScanned : {}", resultBlock.getNumDocsScanned()); LOGGER.debug("TotalDocs : {}", resultBlock.getTotalRawDocs()); LOGGER.debug("Result: {}", resultBlock.getAggregationGroupByResultDataTable()); final AggregationGroupByOperatorService aggregationGroupByOperatorService = new AggregationGroupByOperatorService(_aggregationInfos, brokerRequest.getGroupBy()); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put( new ServerInstance("localhost:0000"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:1111"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:2222"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:3333"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:4444"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:5555"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:6666"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:7777"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:8888"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:9999"), resultBlock.getAggregationGroupByResultDataTable()); final List<Map<String, Serializable>> reducedResults = aggregationGroupByOperatorService.reduceGroupByOperators(instanceResponseMap); final List<JSONObject> jsonResult = aggregationGroupByOperatorService.renderGroupByOperators(reducedResults); LOGGER.debug("Json Result: {}", jsonResult); } @Test public void testInnerSegmentPlanMakerForAggregationGroupByOperatorWithFilter() throws Exception { final BrokerRequest brokerRequest = getAggregationGroupByWithFilterBrokerRequest(); final PlanMaker instancePlanMaker = new InstancePlanMakerImplV1(); final PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); final MAggregationGroupByOperator operator = (MAggregationGroupByOperator) rootPlanNode.run(); final IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) operator.nextBlock(); LOGGER.debug("RunningTime : {}", resultBlock.getTimeUsedMs()); LOGGER.debug("NumDocsScanned : {}", resultBlock.getNumDocsScanned()); LOGGER.debug("TotalDocs : {}", resultBlock.getTotalRawDocs()); LOGGER.debug("Result: {}", resultBlock.getAggregationGroupByResultDataTable()); Assert.assertEquals(resultBlock.getNumDocsScanned(), 582); Assert.assertEquals(resultBlock.getTotalRawDocs(), 10001); final AggregationGroupByOperatorService aggregationGroupByOperatorService = new AggregationGroupByOperatorService(_aggregationInfos, brokerRequest.getGroupBy()); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put( new ServerInstance("localhost:0000"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:1111"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:2222"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:3333"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:4444"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:5555"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:6666"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:7777"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:8888"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:9999"), resultBlock.getAggregationGroupByResultDataTable()); final List<Map<String, Serializable>> reducedResults = aggregationGroupByOperatorService.reduceGroupByOperators(instanceResponseMap); final List<JSONObject> jsonResult = aggregationGroupByOperatorService.renderGroupByOperators(reducedResults); LOGGER.debug("Result: {}", jsonResult); } @Test public void testInterSegmentAggregationGroupByPlanMakerAndRun() throws Exception { final int numSegments = 20; setupSegmentList(numSegments); final PlanMaker instancePlanMaker = new InstancePlanMakerImplV1(); final BrokerRequest brokerRequest = getAggregationGroupByNoFilterBrokerRequest(); final BrokerResponse brokerResponse = getBrokerResponse(instancePlanMaker, brokerRequest); assertBrokerResponse(numSegments, brokerResponse); } private BrokerResponse getBrokerResponse( PlanMaker instancePlanMaker, BrokerRequest brokerRequest) { final ExecutorService executorService = Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker")); final Plan globalPlan = instancePlanMaker.makeInterSegmentPlan( _indexSegmentList, brokerRequest, executorService, 150000); globalPlan.print(); globalPlan.execute(); final DataTable instanceResponse = globalPlan.getInstanceResponse(); LOGGER.debug("Instance Response: {}", instanceResponse); final DefaultReduceService defaultReduceService = new DefaultReduceService(); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); final BrokerResponse brokerResponse = defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.debug("Broker Response: {}", new JSONArray(brokerResponse.getAggregationResults())); LOGGER.debug("Time used : {}", brokerResponse.getTimeUsedMs()); return brokerResponse; } @Test public void testEmptyQueryResultsForInterSegmentAggregationGroupBy() throws Exception { final int numSegments = 20; setupSegmentList(numSegments); final PlanMaker instancePlanMaker = new InstancePlanMakerImplV1(); final BrokerRequest brokerRequest = getAggregationGroupByWithEmptyFilterBrokerRequest(); final BrokerResponse brokerResponse = getBrokerResponse(instancePlanMaker, brokerRequest); assertEmptyBrokerResponse(brokerResponse); } private void assertBrokerResponse(int numSegments, BrokerResponse brokerResponse) throws JSONException { Assert.assertEquals(10001 * numSegments, brokerResponse.getNumDocsScanned()); final int groupSize = 15; assertBrokerResponse(brokerResponse, groupSize); // Assertion on Aggregation Results final List<double[]> aggregationResult = getAggregationResult(numSegments); final List<String[]> groupByResult = getGroupResult(); for (int j = 0; j < _numAggregations; ++j) { final double[] aggResult = aggregationResult.get(j); final String[] groupResult = groupByResult.get(j); for (int i = 0; i < 15; ++i) { Assert.assertEquals( 0, DoubleComparisonUtil.defaultDoubleCompare( aggResult[i], brokerResponse .getAggregationResults() .get(j) .getJSONArray("groupByResult") .getJSONObject(i) .getDouble("value"))); if ((i < 14 && aggResult[i] == aggResult[i + 1]) || (i > 0 && aggResult[i] == aggResult[i - 1])) { // do nothing, as we have multiple groups within same value. } else { Assert.assertEquals( groupResult[i], brokerResponse .getAggregationResults() .get(j) .getJSONArray("groupByResult") .getJSONObject(i) .getString("group")); } } } } private void assertBrokerResponse(BrokerResponse brokerResponse, int groupSize) throws JSONException { Assert.assertEquals(_numAggregations, brokerResponse.getAggregationResults().size()); for (int i = 0; i < _numAggregations; ++i) { Assert.assertEquals( "[\"column11\",\"column10\"]", brokerResponse.getAggregationResults().get(i).getJSONArray("groupByColumns").toString()); Assert.assertEquals( groupSize, brokerResponse.getAggregationResults().get(i).getJSONArray("groupByResult").length()); } // Assertion on Count Assert.assertEquals( "count_star", brokerResponse.getAggregationResults().get(0).getString("function").toString()); Assert.assertEquals( "sum_met_impressionCount", brokerResponse.getAggregationResults().get(1).getString("function").toString()); Assert.assertEquals( "max_met_impressionCount", brokerResponse.getAggregationResults().get(2).getString("function").toString()); Assert.assertEquals( "min_met_impressionCount", brokerResponse.getAggregationResults().get(3).getString("function").toString()); Assert.assertEquals( "avg_met_impressionCount", brokerResponse.getAggregationResults().get(4).getString("function").toString()); Assert.assertEquals( "distinctCount_column12", brokerResponse.getAggregationResults().get(5).getString("function").toString()); } private void assertEmptyBrokerResponse(BrokerResponse brokerResponse) throws JSONException { Assert.assertEquals(0, brokerResponse.getNumDocsScanned()); final int groupSize = 0; assertBrokerResponse(brokerResponse, groupSize); } private static List<double[]> getAggregationResult(int numSegments) { final List<double[]> aggregationResultList = new ArrayList<double[]>(); aggregationResultList.add(getCountResult(numSegments)); aggregationResultList.add(getSumResult(numSegments)); aggregationResultList.add(getMaxResult()); aggregationResultList.add(getMinResult()); aggregationResultList.add(getAvgResult()); aggregationResultList.add(getDistinctCountResult()); return aggregationResultList; } private static List<String[]> getGroupResult() { final List<String[]> groupResults = new ArrayList<String[]>(); groupResults.add(getCountGroupResult()); groupResults.add(getSumGroupResult()); groupResults.add(getMaxGroupResult()); groupResults.add(getMinGroupResult()); groupResults.add(getAvgGroupResult()); groupResults.add(getDistinctCountGroupResult()); return groupResults; } private static double[] getCountResult(int numSegments) { return new double[] { 1450 * numSegments, 620 * numSegments, 517 * numSegments, 422 * numSegments, 365 * numSegments, 340 * numSegments, 321 * numSegments, 296 * numSegments, 286 * numSegments, 273 * numSegments, 271 * numSegments, 268 * numSegments, 234 * numSegments, 210 * numSegments, 208 * numSegments }; } private static String[] getCountGroupResult() { return new String[] { "[\"i\",\"\"]", "[\"D\",\"\"]", "[\"i\",\"CqC\"]", "[\"i\",\"QMl\"]", "[\"i\",\"bVnY\"]", "[\"i\",\"iV\"]", "[\"i\",\"zZe\"]", "[\"i\",\"xDLG\"]", "[\"i\",\"VsKz\"]", "[\"i\",\"mNh\"]", "[\"i\",\"ez\"]", "[\"i\",\"rNcu\"]", "[\"i\",\"EXYv\"]", "[\"i\",\"gpyD\"]", "[\"i\",\"yhq\"]" }; } private static double[] getSumResult(int numSegments) { return new double[] { 194232989695956150000000.00000, 82874083725452570000000.00000, 69188102307666020000000.00000, 57011594945268800000000.00000, 49669069292549060000000.00000, 45658425435674350000000.00000, 42733154649942075000000.00000, 39374565823833550000000.00000, 38376043393352970000000.00000, 36944406922141550000000.00000, 36562112604244086000000.00000, 36141768458849143000000.00000, 31259578136918286000000.00000, 27679187240218786000000.00000, 27524721980723073000000.00000 }; } private static String[] getSumGroupResult() { return new String[] { "[\"i\",\"\"]", "[\"D\",\"\"]", "[\"i\",\"CqC\"]", "[\"i\",\"QMl\"]", "[\"i\",\"bVnY\"]", "[\"i\",\"iV\"]", "[\"i\",\"zZe\"]", "[\"i\",\"xDLG\"]", "[\"i\",\"VsKz\"]", "[\"i\",\"mNh\"]", "[\"i\",\"ez\"]", "[\"i\",\"rNcu\"]", "[\"i\",\"EXYv\"]", "[\"i\",\"yhq\"]", "[\"i\",\"gpyD\"]" }; } private static double[] getMaxResult() { return new double[] { 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000, 8637957270245934100.00000 }; } private static String[] getMaxGroupResult() { return new String[] { "[\"i\",\"yH\"]", "[\"U\",\"mNh\"]", "[\"i\",\"OYMU\"]", "[\"D\",\"opm\"]", "[\"D\",\"Gac\"]", "[\"i\",\"ZQa\"]", "[\"i\",\"gpyD\"]", "[\"D\",\"Pcb\"]", "[\"i\",\"mNh\"]", "[\"U\",\"LjAS\"]", "[\"U\",\"bVnY\"]", "[\"D\",\"aN\"]", "[\"D\",\"iV\"]", "[\"U\",\"Vj\"]", "[\"D\",\"KsKZ\"]" }; } private static double[] getMinResult() { return new double[] { 614819680033322500.00000, 614819680033322500.00000, 614819680033322500.00000, 614819680033322500.00000, 614819680033322500.00000, 614819680033322500.00000, 614819680033322500.00000, 614819680033322500.00000, 1048718684474966140.00000, 1048718684474966140.00000, 1048718684474966140.00000, 3703896352903212000.00000, 3703896352903212000.00000, 3703896352903212000.00000, 3703896352903212000.00000 }; } private static String[] getMinGroupResult() { return new String[] { "[\"D\",\"Gac\"]", "[\"i\",\"mNh\"]", "[\"i\",\"VsKz\"]", "[\"D\",\"\"]", "[\"i\",\"yhq\"]", "[\"D\",\"CqC\"]", "[\"U\",\"\"]", "[\"i\",\"jb\"]", "[\"D\",\"bVnY\"]", "[\"i\",\"\"]", "[\"i\",\"QMl\"]", "[\"i\",\"Pcb\"]", "[\"i\",\"EXYv\"]", "[\"i\",\"bVnY\"]", "[\"i\",\"zZe\"]" }; } private static double[] getAvgResult() { return new double[] { 7768390271561314300.00000, 7215319188094814200.00000, 7105513810764889100.00000, 7094438547504759800.00000, 7004199482369404900.00000, 6991851055242935300.00000, 6987779156890090500.00000, 6973627660796153900.00000, 6970558938737374200.00000, 6964262042984379400.00000, 6912897688920598500.00000, 6906152143309600800.00000, 6888134675143909400.00000, 6880505863259489300.00000, 6878447250928267300.00000 }; } private static String[] getAvgGroupResult() { return new String[] { "[\"U\",\"yhq\"]", "[\"U\",\"mNh\"]", "[\"U\",\"Vj\"]", "[\"U\",\"OYMU\"]", "[\"U\",\"zZe\"]", "[\"U\",\"jb\"]", "[\"D\",\"aN\"]", "[\"U\",\"bVnY\"]", "[\"U\",\"iV\"]", "[\"i\",\"LjAS\"]", "[\"D\",\"xDLG\"]", "[\"U\",\"EXYv\"]", "[\"D\",\"iV\"]", "[\"D\",\"Gac\"]", "[\"D\",\"QMl\"]" }; } private static double[] getDistinctCountResult() { return new double[] {128, 109, 100, 99, 84, 81, 77, 76, 75, 74, 71, 67, 67, 62, 57}; } private static String[] getDistinctCountGroupResult() { return new String[] { "[\"i\",\"\"]", "[\"D\",\"\"]", "[\"i\",\"zZe\"]", "[\"i\",\"QMl\"]", "[\"i\",\"bVnY\"]", "[\"i\",\"iV\"]", "[\"i\",\"VsKz\"]", "[\"i\",\"CqC\"]", "[\"i\",\"EXYv\"]", "[\"i\",\"xDLG\"]", "[\"i\",\"yhq\"]", "[\"U\",\"\"]", "[\"D\",\"EXYv\"]", "[\"D\",\"LjAS\"]", "[\"i\",\"rNcu\"]" }; } private static BrokerRequest getAggregationGroupByNoFilterBrokerRequest() { final BrokerRequest brokerRequest = new BrokerRequest(); setAggregationsInfo(brokerRequest); return brokerRequest; } private static List<AggregationInfo> getAggregationsInfo() { final List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(getCountAggregationInfo()); aggregationsInfo.add(getSumAggregationInfo()); aggregationsInfo.add(getMaxAggregationInfo()); aggregationsInfo.add(getMinAggregationInfo()); aggregationsInfo.add(getAvgAggregationInfo()); aggregationsInfo.add(getDistinctCountAggregationInfo("column12")); return aggregationsInfo; } private static Map<String, DataSource> getDataSourceMap() { final Map<String, DataSource> dataSourceMap = new HashMap<String, DataSource>(); dataSourceMap.put("column11", _indexSegment.getDataSource("column11")); dataSourceMap.put("column12", _indexSegment.getDataSource("column12")); dataSourceMap.put("met_impressionCount", _indexSegment.getDataSource("met_impressionCount")); return dataSourceMap; } private static AggregationInfo getCountAggregationInfo() { final String type = "count"; final Map<String, String> params = new HashMap<String, String>(); params.put("column", "*"); final AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private static AggregationInfo getSumAggregationInfo() { final String type = "sum"; return getAggregationInfo(type); } private static AggregationInfo getAggregationInfo(String type) { final Map<String, String> params = new HashMap<String, String>(); params.put("column", "met_impressionCount"); final AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private static AggregationInfo getMaxAggregationInfo() { final String type = "max"; return getAggregationInfo(type); } private static AggregationInfo getMinAggregationInfo() { final String type = "min"; return getAggregationInfo(type); } private static AggregationInfo getAvgAggregationInfo() { final String type = "avg"; return getAggregationInfo(type); } private static AggregationInfo getDistinctCountAggregationInfo(String dim) { final String type = "distinctCount"; final Map<String, String> params = new HashMap<String, String>(); params.put("column", dim); final AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private static GroupBy getGroupBy() { final GroupBy groupBy = new GroupBy(); final List<String> columns = new ArrayList<String>(); columns.add("column11"); columns.add("column10"); groupBy.setColumns(columns); groupBy.setTopN(15); return groupBy; } private static BrokerRequest getAggregationGroupByWithFilterBrokerRequest() { final BrokerRequest brokerRequest = new BrokerRequest(); setAggregationsInfo(brokerRequest); setFilterQuery(brokerRequest); return brokerRequest; } private static void setAggregationsInfo(BrokerRequest brokerRequest) { final List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(getCountAggregationInfo()); aggregationsInfo.add(getSumAggregationInfo()); aggregationsInfo.add(getMaxAggregationInfo()); aggregationsInfo.add(getMinAggregationInfo()); aggregationsInfo.add(getAvgAggregationInfo()); aggregationsInfo.add(getDistinctCountAggregationInfo("column12")); brokerRequest.setAggregationsInfo(aggregationsInfo); brokerRequest.setGroupBy(getGroupBy()); } private static BrokerRequest setFilterQuery(BrokerRequest brokerRequest) { FilterQueryTree filterQueryTree; final String filterColumn = "column11"; final String filterVal = "U"; generateFilter(brokerRequest, filterColumn, filterVal); return brokerRequest; } private static void generateFilter( BrokerRequest brokerRequest, String filterColumn, String filterVal) { FilterQueryTree filterQueryTree; if (filterColumn.contains(",")) { final String[] filterColumns = filterColumn.split(","); final String[] filterValues = filterVal.split(","); final List<FilterQueryTree> nested = new ArrayList<FilterQueryTree>(); for (int i = 0; i < filterColumns.length; i++) { final List<String> vals = new ArrayList<String>(); vals.add(filterValues[i]); final FilterQueryTree d = new FilterQueryTree(i + 1, filterColumns[i], vals, FilterOperator.EQUALITY, null); nested.add(d); } filterQueryTree = new FilterQueryTree(0, null, null, FilterOperator.AND, nested); } else { final List<String> vals = new ArrayList<String>(); vals.add(filterVal); filterQueryTree = new FilterQueryTree(0, filterColumn, vals, FilterOperator.EQUALITY, null); } RequestUtils.generateFilterFromTree(filterQueryTree, brokerRequest); } private static BrokerRequest getAggregationGroupByWithEmptyFilterBrokerRequest() { final BrokerRequest brokerRequest = new BrokerRequest(); setAggregationsInfo(brokerRequest); setEmptyFilterQuery(brokerRequest); return brokerRequest; } private static BrokerRequest setEmptyFilterQuery(BrokerRequest brokerRequest) { FilterQueryTree filterQueryTree; final String filterColumn = "column11"; final String filterVal = "uuuu"; generateFilter(brokerRequest, filterColumn, filterVal); /* if (filterColumn.contains(",")) { final String[] filterColumns = filterColumn.split(","); final String[] filterValues = filterVal.split(","); final List<FilterQueryTree> nested = new ArrayList<FilterQueryTree>(); for (int i = 0; i < filterColumns.length; i++) { final List<String> vals = new ArrayList<String>(); vals.add(filterValues[i]); final FilterQueryTree d = new FilterQueryTree(i + 1, filterColumns[i], vals, FilterOperator.EQUALITY, null); nested.add(d); } filterQueryTree = new FilterQueryTree(0, null, null, FilterOperator.AND, nested); } else { final List<String> vals = new ArrayList<String>(); vals.add(filterVal); filterQueryTree = new FilterQueryTree(0, filterColumn, vals, FilterOperator.EQUALITY, null); } RequestUtils.generateFilterFromTree(filterQueryTree, brokerRequest); */ return brokerRequest; } }
private Path getDefaultPath() { return FileUtils.getTempDirectory() .toPath() .resolve("Windup") .resolve("windupgraph_javaclasstest_" + RandomStringUtils.randomAlphanumeric(6)); }
/** * 在系统临时目录中创建文件(文件名随机)并返回 * * @author adli 2015年7月19日 下午5:56:56 * @return */ private static File getTmpFile() { File tmpDir = FileUtils.getTempDirectory(); String tmpFileName = (Double.toString(Math.random() * 10000.0D)).replace(".", ""); return new File(tmpDir, tmpFileName); }
private Path getDefaultPath() { return FileUtils.getTempDirectory() .toPath() .resolve("Windup") .resolve("windupgraph_nonnamespacedmavendisc_" + RandomStringUtils.randomAlphanumeric(6)); }
/** * Runs a MAF file through the Oncotator and OMA tools. * * @param inputMAFURL String * @param outputMAFURL String * @throws Exception */ @Override public void oncotateMAF(String inputMAFURL, String outputMAFURL) throws Exception { // sanity check if (inputMAFURL == null || inputMAFURL.length() == 0 || outputMAFURL == null || outputMAFURL.length() == 0) { throw new IllegalArgumentException( "oncotateMAFdownloadFile(): url or urlDestination argument is null..."); } URL inputMAF = new URL(inputMAFURL); URL outputMAF = new URL(outputMAFURL); // determine if we have to call liftover boolean cleanOncotatorInputFile = false; File oncotatorInputFile = new File(inputMAF.getFile()); org.apache.commons.io.LineIterator it = org.apache.commons.io.FileUtils.lineIterator(oncotatorInputFile); it.nextLine(); // skip header String[] parts = it.nextLine().split("\t"); if (parts[3].contains("36") || parts[3].equals("hg18")) { it.close(); File liftoverInputFile = org.apache.commons.io.FileUtils.getFile( org.apache.commons.io.FileUtils.getTempDirectory(), "liftoverInputFile"); org.apache.commons.io.FileUtils.copyFile(oncotatorInputFile, liftoverInputFile); oncotatorInputFile = new File(inputMAF.getFile()); // call lift over if (LOG.isInfoEnabled()) { LOG.info("oncotateMAF(), calling Hg18ToHg19..."); } Hg18ToHg19.driver( liftoverInputFile.getCanonicalPath(), oncotatorInputFile.getCanonicalPath(), getLiftOverBinary(), getLiftOverChain()); org.apache.commons.io.FileUtils.forceDelete(liftoverInputFile); cleanOncotatorInputFile = true; } // create a temp output file from the oncotator File oncotatorOutputFile = org.apache.commons.io.FileUtils.getFile( org.apache.commons.io.FileUtils.getTempDirectory(), "oncotatorOutputFile"); // call oncotator if (LOG.isInfoEnabled()) { LOG.info("oncotateMAF(), calling OncotateTool..."); } OncotateTool.driver( oncotatorInputFile.getCanonicalPath(), oncotatorOutputFile.getCanonicalPath(), true, true, true); // we call OMA here - // we use output from oncotator as input file if (LOG.isInfoEnabled()) { LOG.info("oncotateMAF(), calling MutationAssessorTool..."); } File outputMAFFile = new File(outputMAF.getFile()); outputMAFFile.createNewFile(); MutationAssessorTool.driver( oncotatorOutputFile.getCanonicalPath(), outputMAFFile.getCanonicalPath(), false, true, true); // clean up org.apache.commons.io.FileUtils.forceDelete(oncotatorOutputFile); if (cleanOncotatorInputFile) org.apache.commons.io.FileUtils.forceDelete(oncotatorInputFile); }
public class PlanMakerTest { private static final Logger LOGGER = LoggerFactory.getLogger(PlanMakerTest.class); private static final String LARGE_AVRO_DATA = "data/simpleData2000001.avro"; private static final String SMALL_AVRO_DATA = "data/simpleData200001.avro"; private static File INDEX_DIR = new File(FileUtils.getTempDirectory() + File.separator + "TestPlanMaker"); private static File INDEXES_DIR = new File(FileUtils.getTempDirectory() + File.separator + "TestPlanMakerList"); private static final int COUNT_AGGREGATION_INDEX = 0; private static final int SUM_AGGREGATION_INDEX = 1; private static final int MAX_AGGREGATION_INDEX = 2; private static final int MIN_AGGREGATION_INDEX = 3; private static final int AVG_AGGREGATION_INDEX = 4; private static final int DISTINCT_DIM0_AGGREGATION_INDEX = 5; private static final int DISTINCT_DIM1_AGGREGATION_INDEX = 6; private BrokerRequest _brokerRequest; private IndexSegment _indexSegment; private List<IndexSegment> _indexSegmentList = new ArrayList<>(); @BeforeClass public void setup() { _brokerRequest = getAggregationNoFilterBrokerRequest(); try { setupSegment(); setupSegmentList(20); } catch (Exception e) { e.printStackTrace(); } } @AfterClass public void tearDown() { if (_indexSegment != null) { _indexSegment.destroy(); } for (IndexSegment segment : _indexSegmentList) { segment.destroy(); } _indexSegmentList.clear(); if (INDEX_DIR.exists()) { FileUtils.deleteQuietly(INDEX_DIR); } if (INDEXES_DIR.exists()) { FileUtils.deleteQuietly(INDEXES_DIR); } } private void setupSegment() throws Exception { final String filePath = TestUtils.getFileFromResourceUrl( PlanMakerTest.class.getClassLoader().getResource(LARGE_AVRO_DATA)); if (INDEX_DIR.exists()) { FileUtils.deleteQuietly(INDEX_DIR); } final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns( new File(filePath), INDEX_DIR, "dim1", TimeUnit.DAYS, "test"); final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null); driver.init(config); driver.build(); LOGGER.debug("built at: {}", INDEX_DIR.getAbsolutePath()); final File indexSegmentDir = new File(INDEX_DIR, driver.getSegmentName()); _indexSegment = ColumnarSegmentLoader.load(indexSegmentDir, ReadMode.mmap); } private void setupSegmentList(int numberOfSegments) throws Exception { final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(SMALL_AVRO_DATA)); _indexSegmentList.clear(); if (INDEXES_DIR.exists()) { FileUtils.deleteQuietly(INDEXES_DIR); } INDEXES_DIR.mkdir(); for (int i = 0; i < numberOfSegments; ++i) { final File segmentDir = new File(INDEXES_DIR, "segment_" + i); final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns( new File(filePath), segmentDir, "dim1", TimeUnit.DAYS, "test"); final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null); driver.init(config); driver.build(); LOGGER.debug("built at: {}", segmentDir.getAbsolutePath()); _indexSegmentList.add( ColumnarSegmentLoader.load(new File(segmentDir, driver.getSegmentName()), ReadMode.mmap)); } } @Test public void testInnerSegmentPlanMakerForAggregationNoFilter() { BrokerRequest brokerRequest = getAggregationNoFilterBrokerRequest(); PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) rootPlanNode.run().nextBlock(); for (int i = 0; i < 7; i++) { LOGGER.debug(resultBlock.getAggregationResult().get(i).toString()); } assertEquals(((Number) resultBlock.getAggregationResult().get(0)).longValue(), 2000001L); assertEquals(resultBlock.getAggregationResult().get(1), 2000001000000.0); assertEquals(resultBlock.getAggregationResult().get(2), 2000000.0); assertEquals(resultBlock.getAggregationResult().get(3), 0.0); assertEquals( Double.parseDouble(resultBlock.getAggregationResult().get(4).toString()), 1000000.0); assertEquals(((IntOpenHashSet) resultBlock.getAggregationResult().get(5)).size(), 10); assertEquals(((IntOpenHashSet) resultBlock.getAggregationResult().get(6)).size(), 100); } @Test public void testInnerSegmentPlanMakerForAggregationWithFilter() { BrokerRequest brokerRequest = getAggregationWithFilterBrokerRequest(); PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); } @Test public void testInnerSegmentPlanMakerForSelectionNoFilter() { BrokerRequest brokerRequest = getSelectionNoFilterBrokerRequest(); PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) rootPlanNode.run().nextBlock(); PriorityQueue<Serializable[]> retPriorityQueue = (PriorityQueue<Serializable[]>) resultBlock.getSelectionResult(); while (!retPriorityQueue.isEmpty()) { Serializable[] row = retPriorityQueue.poll(); LOGGER.debug(Arrays.toString(row)); assertEquals(row[0], 9); assertEquals(row[1], 99); } } @Test public void testInnerSegmentPlanMakerForSelectionNoFilterNoOrdering() { BrokerRequest brokerRequest = getSelectionNoFilterBrokerRequest(); brokerRequest.getSelections().setSelectionSortSequence(null); PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); // USelectionOperator operator = (USelectionOperator) rootPlanNode.run(); MSelectionOnlyOperator operator = (MSelectionOnlyOperator) rootPlanNode.run(); IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) operator.nextBlock(); ArrayList<Serializable[]> retList = (ArrayList<Serializable[]>) resultBlock.getSelectionResult(); int i = 0; for (Serializable[] row : retList) { LOGGER.debug(Arrays.toString(row)); assertEquals(row[0], (i % 10)); assertEquals(row[1], i); assertEquals(row[2], i); i++; } } @Test public void testInnerSegmentPlanMakerForSelectionWithFilter() { BrokerRequest brokerRequest = getSelectionWithFilterBrokerRequest(); PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); } @Test public void testInnerSegmentPlanMakerForAggregationGroupByNoFilter() { BrokerRequest brokerRequest = getAggregationGroupByNoFilterBrokerRequest(); PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) rootPlanNode.run().nextBlock(); LOGGER.debug("RunningTime: {}", resultBlock.getTimeUsedMs()); LOGGER.debug("NumDocsScanned: {}", resultBlock.getNumDocsScanned()); LOGGER.debug("TotalDocs: {}", resultBlock.getTotalRawDocs()); List<Map<String, Serializable>> combinedGroupByResult = resultBlock.getAggregationGroupByOperatorResult(); Map<String, Serializable> singleGroupByResult = combinedGroupByResult.get(COUNT_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(200001, ((Number) resultList).longValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(200000, ((Number) resultList).longValue()); } } singleGroupByResult = combinedGroupByResult.get(SUM_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedSumValue = ((Double.parseDouble(keyString) + 2000000 + Double.parseDouble(keyString)) * 200001) / 2; assertEquals(expectedSumValue, ((Double) resultList).doubleValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedSumValue = (((Double.parseDouble(keyString) + 2000000) - 10) + Double.parseDouble(keyString)) * 100000; assertEquals(expectedSumValue, ((Double) resultList).doubleValue()); } } singleGroupByResult = combinedGroupByResult.get(MAX_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(2000000 + Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals( (2000000 - 10) + Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } } singleGroupByResult = combinedGroupByResult.get(MIN_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } } singleGroupByResult = combinedGroupByResult.get(AVG_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedAvgValue = ((Double.parseDouble(keyString) + 2000000 + Double.parseDouble(keyString)) * 200001) / 2 / 200001; assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString()))); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedAvgValue = ((((Double.parseDouble(keyString) + 2000000) - 10) + Double.parseDouble(keyString)) * 100000) / 200000; assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString()))); } } singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM0_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(1, ((IntOpenHashSet) resultList).size()); } singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM1_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(10, ((IntOpenHashSet) resultList).size()); } } @Test public void testInnerSegmentPlanMakerForAggregationGroupByWithFilter() { BrokerRequest brokerRequest = getAggregationGroupByWithFilterBrokerRequest(); brokerRequest.getGroupBy().getColumns().clear(); brokerRequest.getGroupBy().getColumns().add("dim0"); brokerRequest.getGroupBy().getColumns().add("dim1"); PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) rootPlanNode.run().nextBlock(); LOGGER.debug("RunningTime: {}", resultBlock.getTimeUsedMs()); LOGGER.debug("NumDocsScanned: {}", resultBlock.getNumDocsScanned()); LOGGER.debug("TotalDocs: {}", resultBlock.getTotalRawDocs()); List<Map<String, Serializable>> combinedGroupByResult = resultBlock.getAggregationGroupByOperatorResult(); for (int i = 0; i < combinedGroupByResult.size(); ++i) { LOGGER.debug("function: {}", brokerRequest.getAggregationsInfo().get(i)); for (String keyString : combinedGroupByResult.get(i).keySet()) { LOGGER.debug( "grouped key: {}, value: {}", keyString, combinedGroupByResult.get(i).get(keyString)); } } } @Test public void testInterSegmentAggregationPlanMaker() { PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); BrokerRequest brokerRequest = _brokerRequest.deepCopy(); brokerRequest.setSelections(null); brokerRequest.setSelectionsIsSet(false); ExecutorService executorService = Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker")); Plan globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); brokerRequest = setFilterQuery(brokerRequest); globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); } private List<SegmentDataManager> makeSegMgrList(List<IndexSegment> indexSegmentList) { List<SegmentDataManager> segMgrList = new ArrayList<>(indexSegmentList.size()); for (IndexSegment segment : indexSegmentList) { segMgrList.add(new OfflineSegmentDataManager(segment)); } return segMgrList; } @Test public void testInterSegmentAggregationPlanMakerAndRun() { PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); BrokerRequest brokerRequest = _brokerRequest.deepCopy(); ExecutorService executorService = Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker")); Plan globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); globalPlan.execute(); DataTable instanceResponse = globalPlan.getInstanceResponse(); LOGGER.debug(Long.toString(instanceResponse.getLong(0, 0))); LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 1))); LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 2))); LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 3))); LOGGER.debug(instanceResponse.getObject(0, 4).toString()); LOGGER.debug(instanceResponse.getObject(0, 5).toString()); LOGGER.debug(instanceResponse.getObject(0, 6).toString()); LOGGER.debug("Query time: {}", instanceResponse.getMetadata().get("timeUsedMs")); assertEquals(200001L * _indexSegmentList.size(), instanceResponse.getLong(0, 0)); assertEquals(20000100000.0 * _indexSegmentList.size(), instanceResponse.getDouble(0, 1)); assertEquals(200000.0, instanceResponse.getDouble(0, 2)); assertEquals(0.0, instanceResponse.getDouble(0, 3)); assertEquals(100000.0, Double.parseDouble(instanceResponse.getObject(0, 4).toString())); assertEquals(10, ((IntOpenHashSet) instanceResponse.getObject(0, 5)).size()); assertEquals(100, ((IntOpenHashSet) instanceResponse.getObject(0, 6)).size()); DefaultReduceService reduceService = new DefaultReduceService(); Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse); BrokerResponseJSON brokerResponse = reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.debug(brokerResponse.getAggregationResults().toString()); } @Test public void testInterSegmentAggregationGroupByPlanMakerAndRun() { PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); BrokerRequest brokerRequest = getAggregationGroupByNoFilterBrokerRequest(); ExecutorService executorService = Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker")); Plan globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); globalPlan.execute(); DataTable instanceResponse = globalPlan.getInstanceResponse(); LOGGER.debug(instanceResponse.toString()); List<DataTable> instanceResponseList = new ArrayList<DataTable>(); instanceResponseList.add(instanceResponse); List<Map<String, Serializable>> combinedGroupByResult = AggregationGroupByOperatorService.transformDataTableToGroupByResult(instanceResponse); Map<String, Serializable> singleGroupByResult = combinedGroupByResult.get(COUNT_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(400020, ((Number) resultList).longValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(400000, ((Number) resultList).longValue()); } } singleGroupByResult = combinedGroupByResult.get(SUM_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedSumValue = (((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001) / 2) * 20; assertEquals(expectedSumValue, ((Double) resultList).doubleValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedSumValue = (((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString)) * 10000 * 20; assertEquals(expectedSumValue, ((Double) resultList).doubleValue()); } } singleGroupByResult = combinedGroupByResult.get(MAX_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(200000 + Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals( (200000 - 10) + Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } } singleGroupByResult = combinedGroupByResult.get(MIN_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } } singleGroupByResult = combinedGroupByResult.get(AVG_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedAvgValue = ((((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001) / 2) * 20) / 400020; assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString()))); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedAvgValue = ((((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString)) * 10000 * 20) / 400000; assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString()))); } } singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM0_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); int expectedAvgValue = 1; assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size()); } singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM1_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); int expectedAvgValue = 10; assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size()); } DefaultReduceService defaultReduceService = new DefaultReduceService(); Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); BrokerResponseJSON brokerResponse = defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.debug(new JSONArray(brokerResponse.getAggregationResults()).toString()); LOGGER.debug("Time used: {}", brokerResponse.getTimeUsedMs()); } @Test public void testInterSegmentSelectionPlanMaker() throws JSONException { PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); BrokerRequest brokerRequest = _brokerRequest.deepCopy(); brokerRequest.setAggregationsInfo(null); brokerRequest.setAggregationsInfoIsSet(false); brokerRequest.setSelections(getSelectionQuery()); brokerRequest.getSelections().setOffset(0); brokerRequest.getSelections().setSize(20); ExecutorService executorService = Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker")); Plan globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); brokerRequest = setFilterQuery(brokerRequest); globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); globalPlan.execute(); DataTable instanceResponse = globalPlan.getInstanceResponse(); DefaultReduceService defaultReduceService = new DefaultReduceService(); Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse); BrokerResponseJSON brokerResponse = defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.debug(brokerResponse.getSelectionResults().toString()); LOGGER.debug("Time used: {}", brokerResponse.getTimeUsedMs()); LOGGER.debug(brokerResponse.toString()); JSONArray selectionResultsArray = brokerResponse.getSelectionResults().getJSONArray("results"); for (int j = 0; j < selectionResultsArray.length(); ++j) { LOGGER.debug(selectionResultsArray.getJSONArray(j).toString()); assertEquals(selectionResultsArray.getJSONArray(j).getInt(0), 1); assertEquals(selectionResultsArray.getJSONArray(j).getInt(1), 91); assertEquals(selectionResultsArray.getJSONArray(j).getInt(2) % 100, 91); } } @Test public void testInterSegmentSelectionNoOrderingPlanMaker() throws JSONException { PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); BrokerRequest brokerRequest = _brokerRequest.deepCopy(); brokerRequest.setAggregationsInfo(null); brokerRequest.setAggregationsInfoIsSet(false); brokerRequest.setSelections(getSelectionQuery()); brokerRequest.getSelections().setSelectionSortSequence(null); brokerRequest.getSelections().setOffset(0); brokerRequest.getSelections().setSize(20); ExecutorService executorService = Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker")); Plan globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); brokerRequest = setFilterQuery(brokerRequest); globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); globalPlan.execute(); DataTable instanceResponse = globalPlan.getInstanceResponse(); DefaultReduceService defaultReduceService = new DefaultReduceService(); Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse); BrokerResponseJSON brokerResponse = defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.debug(brokerResponse.getSelectionResults().toString()); LOGGER.debug("TimeUsedMs: {}", brokerResponse.getTimeUsedMs()); LOGGER.debug(brokerResponse.toString()); JSONArray selectionResultsArray = brokerResponse.getSelectionResults().getJSONArray("results"); for (int j = 0; j < selectionResultsArray.length(); ++j) { assertEquals(selectionResultsArray.getJSONArray(j).getInt(0), 1); } } private static BrokerRequest getAggregationNoFilterBrokerRequest() { BrokerRequest brokerRequest = new BrokerRequest(); List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(getCountAggregationInfo()); aggregationsInfo.add(getSumAggregationInfo()); aggregationsInfo.add(getMaxAggregationInfo()); aggregationsInfo.add(getMinAggregationInfo()); aggregationsInfo.add(getAvgAggregationInfo()); aggregationsInfo.add(getDistinctCountDim0AggregationInfo()); aggregationsInfo.add(getDistinctCountDim1AggregationInfo()); brokerRequest.setAggregationsInfo(aggregationsInfo); return brokerRequest; } private static BrokerRequest getAggregationWithFilterBrokerRequest() { BrokerRequest brokerRequest = new BrokerRequest(); List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(getCountAggregationInfo()); aggregationsInfo.add(getSumAggregationInfo()); aggregationsInfo.add(getMaxAggregationInfo()); aggregationsInfo.add(getMinAggregationInfo()); aggregationsInfo.add(getAvgAggregationInfo()); aggregationsInfo.add(getDistinctCountDim0AggregationInfo()); aggregationsInfo.add(getDistinctCountDim1AggregationInfo()); brokerRequest.setAggregationsInfo(aggregationsInfo); brokerRequest = setFilterQuery(brokerRequest); return brokerRequest; } private static BrokerRequest getAggregationGroupByNoFilterBrokerRequest() { BrokerRequest brokerRequest = new BrokerRequest(); List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(getCountAggregationInfo()); aggregationsInfo.add(getSumAggregationInfo()); aggregationsInfo.add(getMaxAggregationInfo()); aggregationsInfo.add(getMinAggregationInfo()); aggregationsInfo.add(getAvgAggregationInfo()); aggregationsInfo.add(getDistinctCountDim0AggregationInfo()); aggregationsInfo.add(getDistinctCountDim1AggregationInfo()); brokerRequest.setAggregationsInfo(aggregationsInfo); brokerRequest.setGroupBy(getGroupBy()); return brokerRequest; } private static BrokerRequest getAggregationGroupByWithFilterBrokerRequest() { BrokerRequest brokerRequest = new BrokerRequest(); List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(getCountAggregationInfo()); aggregationsInfo.add(getSumAggregationInfo()); aggregationsInfo.add(getMaxAggregationInfo()); aggregationsInfo.add(getMinAggregationInfo()); aggregationsInfo.add(getAvgAggregationInfo()); aggregationsInfo.add(getDistinctCountDim0AggregationInfo()); aggregationsInfo.add(getDistinctCountDim1AggregationInfo()); brokerRequest.setAggregationsInfo(aggregationsInfo); brokerRequest.setGroupBy(getGroupBy()); brokerRequest = setFilterQuery(brokerRequest); return brokerRequest; } private static BrokerRequest getSelectionNoFilterBrokerRequest() { BrokerRequest brokerRequest = new BrokerRequest(); brokerRequest.setSelections(getSelectionQuery()); return brokerRequest; } private static BrokerRequest getSelectionWithFilterBrokerRequest() { BrokerRequest brokerRequest = new BrokerRequest(); brokerRequest.setSelections(getSelectionQuery()); brokerRequest = setFilterQuery(brokerRequest); return brokerRequest; } private static BrokerRequest setFilterQuery(BrokerRequest brokerRequest) { FilterQueryTree filterQueryTree; String filterColumn = "dim0"; String filterVal = "1"; if (filterColumn.contains(",")) { String[] filterColumns = filterColumn.split(","); String[] filterValues = filterVal.split(","); List<FilterQueryTree> nested = new ArrayList<FilterQueryTree>(); for (int i = 0; i < filterColumns.length; i++) { List<String> vals = new ArrayList<String>(); vals.add(filterValues[i]); FilterQueryTree d = new FilterQueryTree(i + 1, filterColumns[i], vals, FilterOperator.EQUALITY, null); nested.add(d); } filterQueryTree = new FilterQueryTree(0, null, null, FilterOperator.AND, nested); } else { List<String> vals = new ArrayList<String>(); vals.add(filterVal); filterQueryTree = new FilterQueryTree(0, filterColumn, vals, FilterOperator.EQUALITY, null); } RequestUtils.generateFilterFromTree(filterQueryTree, brokerRequest); return brokerRequest; } private static Selection getSelectionQuery() { Selection selection = new Selection(); selection.setOffset(10); selection.setSize(10); List<String> selectionColumns = new ArrayList<String>(); selectionColumns.add("dim0"); selectionColumns.add("dim1"); selectionColumns.add("met"); selection.setSelectionColumns(selectionColumns); List<SelectionSort> selectionSortSequence = new ArrayList<SelectionSort>(); SelectionSort selectionSort = new SelectionSort(); selectionSort.setColumn("dim0"); selectionSort.setIsAsc(false); selectionSortSequence.add(selectionSort); selectionSort = new SelectionSort(); selectionSort.setColumn("dim1"); selectionSort.setIsAsc(false); selectionSortSequence.add(selectionSort); selection.setSelectionSortSequence(selectionSortSequence); return selection; } private static AggregationInfo getCountAggregationInfo() { String type = "count"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "met"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private static AggregationInfo getSumAggregationInfo() { String type = "sum"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "met"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private static AggregationInfo getMaxAggregationInfo() { String type = "max"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "met"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private static AggregationInfo getMinAggregationInfo() { String type = "min"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "met"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private static AggregationInfo getAvgAggregationInfo() { String type = "avg"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "met"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private static AggregationInfo getDistinctCountDim0AggregationInfo() { String type = "distinctCount"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "dim0"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private static AggregationInfo getDistinctCountDim1AggregationInfo() { String type = "distinctCount"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "dim1"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private static GroupBy getGroupBy() { GroupBy groupBy = new GroupBy(); List<String> columns = new ArrayList<String>(); columns.add("dim0"); groupBy.setColumns(columns); groupBy.setTopN(15); return groupBy; } public static void main(String[] args) throws FileNotFoundException { PrintWriter jsonFilePW = new PrintWriter("/tmp/simpleData200001.json"); for (int i = 0; i < 200001; ++i) { String s = "{\"dim0\":" + (i % 10) + ","; s += "\"dim1\":" + (i % 100) + ","; s += "\"met\":" + i + "}"; jsonFilePW.println(s); } jsonFilePW.close(); } }
@Test public void testJavaClassCondition() throws IOException, InstantiationException, IllegalAccessException { try (GraphContext context = factory.create(getDefaultPath())) { final String inputDir = "src/test/resources/org/jboss/windup/rules/java"; final Path outputPath = Paths.get( FileUtils.getTempDirectory().toString(), "windup_" + RandomStringUtils.randomAlphanumeric(6)); FileUtils.deleteDirectory(outputPath.toFile()); Files.createDirectories(outputPath); // Fill the graph with test data. ProjectModel pm = context.getFramed().addVertex(null, ProjectModel.class); pm.setName("Main Project"); // Create FileModel for $inputDir FileModel inputPathFrame = context.getFramed().addVertex(null, FileModel.class); inputPathFrame.setFilePath(inputDir); inputPathFrame.setProjectModel(pm); pm.addFileModel(inputPathFrame); // Set project.rootFileModel to inputPath pm.setRootFileModel(inputPathFrame); // Create FileModel for $inputDir/HintsClassificationsTest.java FileModel fileModel = context.getFramed().addVertex(null, FileModel.class); fileModel.setFilePath(inputDir + "/JavaHintsClassificationsTest.java"); fileModel.setProjectModel(pm); pm.addFileModel(fileModel); // Create FileModel for $inputDir/JavaClassTest.java fileModel = context.getFramed().addVertex(null, FileModel.class); fileModel.setFilePath(inputDir + "/JavaClassTest.java"); fileModel.setProjectModel(pm); pm.addFileModel(fileModel); context.getGraph().getBaseGraph().commit(); final WindupConfiguration processorConfig = new WindupConfiguration().setOutputDirectory(outputPath); processorConfig.setRuleProviderFilter( new RuleProviderWithDependenciesPredicate(TestJavaClassTestRuleProvider.class)); processorConfig .setGraphContext(context) .setRuleProviderFilter( new RuleProviderWithDependenciesPredicate(TestJavaClassTestRuleProvider.class)); processorConfig.setInputPath(Paths.get(inputDir)); processorConfig.setOutputDirectory(outputPath); processorConfig.setOptionValue(ScanPackagesOption.NAME, Collections.singletonList("")); processor.execute(processorConfig); GraphService<JavaTypeReferenceModel> typeRefService = new GraphService<>(context, JavaTypeReferenceModel.class); Iterable<JavaTypeReferenceModel> typeReferences = typeRefService.findAll(); Assert.assertTrue(typeReferences.iterator().hasNext()); Assert.assertEquals(3, provider.getFirstRuleMatchCount()); Assert.assertEquals(1, provider.getSecondRuleMatchCount()); } }
public class QueryExecutorTest { private final String SMALL_AVRO_DATA = "data/simpleData200001.avro"; private static File INDEXES_DIR = new File(FileUtils.getTempDirectory() + File.separator + "TestQueryExecutorList"); private List<IndexSegment> _indexSegmentList = new ArrayList<IndexSegment>(); private static ServerQueryExecutorV1Impl _queryExecutor; private static final Logger LOGGER = LoggerFactory.getLogger(QueryExecutorTest.class); public static final String PINOT_PROPERTIES = "pinot.properties"; @BeforeClass public void setup() throws Exception { TableDataManagerProvider.setServerMetrics(new ServerMetrics(new MetricsRegistry())); File confDir = new File(QueryExecutorTest.class.getClassLoader().getResource("conf").toURI()); setupSegmentList(2); // ServerBuilder serverBuilder = new ServerBuilder(confDir.getAbsolutePath()); String configFilePath = confDir.getAbsolutePath(); // build _serverConf PropertiesConfiguration serverConf = new PropertiesConfiguration(); serverConf.setDelimiterParsingDisabled(false); serverConf.load(new File(configFilePath, PINOT_PROPERTIES)); FileBasedInstanceDataManager instanceDataManager = FileBasedInstanceDataManager.getInstanceDataManager(); instanceDataManager.init( new FileBasedInstanceDataManagerConfig(serverConf.subset("pinot.server.instance"))); instanceDataManager.start(); for (int i = 0; i < 2; ++i) { instanceDataManager.getTableDataManager("midas"); instanceDataManager.getTableDataManager("midas").addSegment(_indexSegmentList.get(i)); } _queryExecutor = new ServerQueryExecutorV1Impl(); _queryExecutor.init( serverConf.subset("pinot.server.query.executor"), instanceDataManager, new ServerMetrics(new MetricsRegistry())); } @AfterClass public void tearDown() { if (INDEXES_DIR.exists()) { FileUtils.deleteQuietly(INDEXES_DIR); } for (IndexSegment segment : _indexSegmentList) { segment.destroy(); } _indexSegmentList.clear(); } private void setupSegmentList(int numberOfSegments) throws Exception { final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(SMALL_AVRO_DATA)); _indexSegmentList.clear(); if (INDEXES_DIR.exists()) { FileUtils.deleteQuietly(INDEXES_DIR); } INDEXES_DIR.mkdir(); for (int i = 0; i < numberOfSegments; ++i) { final File segmentDir = new File(INDEXES_DIR, "segment_" + i); final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns( new File(filePath), segmentDir, "dim" + i, TimeUnit.DAYS, "midas"); config.setSegmentNamePostfix(String.valueOf(i)); final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null); driver.init(config); driver.build(); File parent = new File(INDEXES_DIR, "segment_" + String.valueOf(i)); String segmentName = parent.list()[0]; _indexSegmentList.add( ColumnarSegmentLoader.load(new File(parent, segmentName), ReadMode.mmap)); System.out.println("built at : " + segmentDir.getAbsolutePath()); } } @Test public void testCountQuery() { BrokerRequest brokerRequest = getCountQuery(); QuerySource querySource = new QuerySource(); querySource.setTableName("midas"); brokerRequest.setQuerySource(querySource); InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); for (IndexSegment segment : _indexSegmentList) { instanceRequest.getSearchSegments().add(segment.getSegmentName()); } try { DataTable instanceResponse = _queryExecutor.processQuery(instanceRequest); LOGGER.info("InstanceResponse is " + instanceResponse.getLong(0, 0)); Assert.assertEquals(instanceResponse.getLong(0, 0), 400002L); LOGGER.info( "Time used for instanceResponse is " + instanceResponse.getMetadata().get("timeUsedMs")); } catch (Exception e) { e.printStackTrace(); // Should never happen Assert.assertEquals(true, false); } } @Test public void testSumQuery() { BrokerRequest brokerRequest = getSumQuery(); QuerySource querySource = new QuerySource(); querySource.setTableName("midas"); brokerRequest.setQuerySource(querySource); InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); for (IndexSegment segment : _indexSegmentList) { instanceRequest.getSearchSegments().add(segment.getSegmentName()); } try { DataTable instanceResponse = _queryExecutor.processQuery(instanceRequest); LOGGER.info("InstanceResponse is " + instanceResponse.getDouble(0, 0)); Assert.assertEquals(instanceResponse.getDouble(0, 0), 40000200000.0); LOGGER.info( "Time used for instanceResponse is " + instanceResponse.getMetadata().get("timeUsedMs")); } catch (Exception e) { e.printStackTrace(); // Should never happen Assert.assertEquals(true, false); } } @Test public void testMaxQuery() { BrokerRequest brokerRequest = getMaxQuery(); QuerySource querySource = new QuerySource(); querySource.setTableName("midas"); brokerRequest.setQuerySource(querySource); InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); for (IndexSegment segment : _indexSegmentList) { instanceRequest.getSearchSegments().add(segment.getSegmentName()); } try { DataTable instanceResponse = _queryExecutor.processQuery(instanceRequest); LOGGER.info("InstanceResponse is " + instanceResponse.getDouble(0, 0)); Assert.assertEquals(instanceResponse.getDouble(0, 0), 200000.0); LOGGER.info( "Time used for instanceResponse is " + instanceResponse.getMetadata().get("timeUsedMs")); } catch (Exception e) { e.printStackTrace(); // Should never happen Assert.assertEquals(true, false); } } @Test public void testMinQuery() { BrokerRequest brokerRequest = getMinQuery(); QuerySource querySource = new QuerySource(); querySource.setTableName("midas"); brokerRequest.setQuerySource(querySource); InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); for (IndexSegment segment : _indexSegmentList) { instanceRequest.getSearchSegments().add(segment.getSegmentName()); } try { DataTable instanceResponse = _queryExecutor.processQuery(instanceRequest); LOGGER.info("InstanceResponse is " + instanceResponse.getDouble(0, 0)); Assert.assertEquals(instanceResponse.getDouble(0, 0), 0.0); LOGGER.info( "Time used for instanceResponse is " + instanceResponse.getMetadata().get("timeUsedMs")); } catch (Exception e) { e.printStackTrace(); // Should never happen Assert.assertEquals(true, false); } } private BrokerRequest getCountQuery() { BrokerRequest query = new BrokerRequest(); AggregationInfo aggregationInfo = getCountAggregationInfo(); List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(aggregationInfo); query.setAggregationsInfo(aggregationsInfo); FilterQuery filterQuery = getFilterQuery(); query.setFilterQuery(filterQuery); return query; } private BrokerRequest getSumQuery() { BrokerRequest query = new BrokerRequest(); AggregationInfo aggregationInfo = getSumAggregationInfo(); List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(aggregationInfo); query.setAggregationsInfo(aggregationsInfo); FilterQuery filterQuery = getFilterQuery(); query.setFilterQuery(filterQuery); return query; } private BrokerRequest getMaxQuery() { BrokerRequest query = new BrokerRequest(); AggregationInfo aggregationInfo = getMaxAggregationInfo(); List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(aggregationInfo); query.setAggregationsInfo(aggregationsInfo); FilterQuery filterQuery = getFilterQuery(); query.setFilterQuery(filterQuery); return query; } private BrokerRequest getMinQuery() { BrokerRequest query = new BrokerRequest(); AggregationInfo aggregationInfo = getMinAggregationInfo(); List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(aggregationInfo); query.setAggregationsInfo(aggregationsInfo); FilterQuery filterQuery = getFilterQuery(); query.setFilterQuery(filterQuery); return query; } private FilterQuery getFilterQuery() { FilterQuery filterQuery = new FilterQuery(); return null; } private AggregationInfo getCountAggregationInfo() { String type = "count"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "met"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private AggregationInfo getSumAggregationInfo() { String type = "sum"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "met"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private AggregationInfo getMaxAggregationInfo() { String type = "max"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "met"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } private AggregationInfo getMinAggregationInfo() { String type = "min"; Map<String, String> params = new HashMap<String, String>(); params.put("column", "met"); AggregationInfo aggregationInfo = new AggregationInfo(); aggregationInfo.setAggregationType(type); aggregationInfo.setAggregationParams(params); return aggregationInfo; } }