@BeforeClass public void setup() throws NamingException, AnalyticsException, IOException { GenericUtils.clearGlobalCustomDataSourceRepo(); System.setProperty( GenericUtils.WSO2_ANALYTICS_CONF_DIRECTORY_SYS_PROP, "src/test/resources/conf1"); AnalyticsServiceHolder.setHazelcastInstance(null); AnalyticsServiceHolder.setAnalyticsClusterManager(new AnalyticsClusterManagerImpl()); System.setProperty(AnalyticsServiceHolder.FORCE_INDEXING_ENV_PROP, Boolean.TRUE.toString()); this.service = ServiceHolder.getAnalyticsDataService(); ServiceHolder.setAnalyticskExecutor( new SparkAnalyticsExecutor("localhost", 0, "src/test/resources/conf1")); ServiceHolder.getAnalyticskExecutor().startSparkServer("src/test/resources/conf1"); }
private void updateIncProcessingTS() { try { long existingIncTS = ServiceHolder.getIncrementalMetaStore() .getLastProcessedTimestamp(this.tenantId, this.incID, false); if (existingIncTS < this.incMaxTS) { ServiceHolder.getIncrementalMetaStore() .setLastProcessedTimestamp(this.tenantId, this.incID, this.incMaxTS, false); } } catch (AnalyticsException e) { throw new RuntimeException(e.getMessage(), e); } }
@Override public Partition[] getPartitions() { AnalyticsDataResponse resp; try { resp = ServiceHolder.getAnalyticsDataService() .get( this.tenantId, this.tableName, computePartitions(), this.allColumns, timeFrom, timeTo, 0, -1); } catch (AnalyticsException e) { throw new RuntimeException(e.getMessage(), e); } RecordGroup[] rgs = resp.getRecordGroups(); Partition[] result = new Partition[rgs.length]; for (int i = 0; i < result.length; i++) { result[i] = new AnalyticsPartition(resp.getRecordStoreName(), rgs[i], i); } return result; }
@SuppressWarnings({"rawtypes", "unchecked"}) @Override public scala.collection.Iterator<Row> compute(Partition split, TaskContext context) { AnalyticsPartition partition = (AnalyticsPartition) split; try { Iterator<Record> recordsItr = ServiceHolder.getAnalyticsDataService() .readRecords(partition.getRecordStoreName(), partition.getRecordGroup()); return new InterruptibleIterator( context, asScalaIterator( new RowRecordIteratorAdaptor(recordsItr, this.tenantId, this.incEnable, this.incID))); } catch (AnalyticsException e) { throw new RuntimeException(e.getMessage(), e); } }
private int computePartitions() throws AnalyticsException { if (ServiceHolder.getAnalyticskExecutor() != null) { return ServiceHolder.getAnalyticskExecutor().getNumPartitionsHint(); } return AnalyticsConstants.SPARK_DEFAULT_PARTITION_COUNT; }
@AfterClass public void done() throws NamingException, AnalyticsException, IOException { ServiceHolder.getAnalyticskExecutor().stop(); this.service.destroy(); System.clearProperty(AnalyticsServiceHolder.FORCE_INDEXING_ENV_PROP); }
/** @return the initialized {@link JavaSparkContext} */ public JavaSparkContext getJavaSparkContext() { return ServiceHolder.getJavaSparkContext(); }