public int enlargeStream( String streamName, List<ColumnNameTypeValue> columns, Boolean raiseException) throws ServiceException { int addedColumns = 0; StreamDefinition streamMetaData = siddhiManager.getStreamDefinition(streamName); for (ColumnNameTypeValue columnNameTypeValue : columns) { if (!SiddhiUtils.columnAlreadyExistsInStream( columnNameTypeValue.getColumn(), streamMetaData)) { addedColumns++; // JPFM -- Updating the columns in streamStatusDao streamStatusDao.addColumn(streamName, columnNameTypeValue); streamMetaData.attribute( columnNameTypeValue.getColumn(), getSiddhiType(columnNameTypeValue.getType())); } else { if (raiseException) { throw new ServiceException( String.format( "Alter stream error, Column %s already " + "exists.", columnNameTypeValue.getColumn())); } } } return addedColumns; }
@Test public void testStreamDefinitionConversion() throws MalformedStreamDefinitionException { StreamDefinition streamDef = new StreamDefinition("stockStream", "1.1.0"); List<Attribute> meta = new ArrayList<Attribute>(1); meta.add(new Attribute("symbol", AttributeType.STRING)); List<Attribute> payload = new ArrayList<Attribute>(1); payload.add(new Attribute("price", AttributeType.DOUBLE)); streamDef.setMetaData(meta); streamDef.setPayloadData(payload); org.wso2.siddhi.query.api.definition.StreamDefinition siddhiDefinition = null; try { siddhiDefinition = EventProcessorUtil.convertToSiddhiStreamDefinition(streamDef, "stockStream"); } catch (EventStreamConfigurationException e) { log.error(e); } Assert.assertEquals(siddhiDefinition.getAttributeList().size(), 2); log.info(siddhiDefinition); /* StreamDefinition databrigeDefinition = EventProcessorUtil.convertToDatabridgeStreamDefinition(siddhiDefinition, "stockStream/1.1.0"); Assert.assertEquals(databrigeDefinition.getPayloadData().size(), 2); log.info(databrigeDefinition); */ }
public void createInternalStream(String streamName, List<ColumnNameTypeValue> columns) { StreamDefinition newStream = QueryFactory.createStreamDefinition().name(streamName); for (ColumnNameTypeValue column : columns) { newStream.attribute(column.getColumn(), getSiddhiType(column.getType())); } siddhiManager.defineStream(newStream); streamStatusDao.createInferredStream(streamName, columns); }
public String addQuery(String streamName, String queryString) { String queryId = siddhiManager.addQuery(queryString); streamStatusDao.addQuery(streamName, queryId, queryString); for (StreamDefinition streamDefinition : siddhiManager.getStreamDefinitions()) { // XXX refactor to obtain exactly siddhi inferred streams. streamStatusDao.createInferredStream( streamDefinition.getStreamId(), castToColumnNameTypeValue(streamDefinition.getAttributeList())); } return queryId; }
@Test public void Test2() throws SiddhiParserException { StreamDefinition streamDefinition = SiddhiCompiler.parseStreamDefinition( "define stream `define` ( `string` string, price int, volume float );"); Assert.assertEquals( StreamDefinition.id("define") .attribute("string", Attribute.Type.STRING) .attribute("price", Attribute.Type.INT) .attribute("volume", Attribute.Type.FLOAT) .toString(), streamDefinition.toString()); }
@Override public void run() { TCPEventPublisher tcpEventPublisher = null; try { tcpEventPublisher = new TCPEventPublisher("localhost:7612", isSynchronous, null); tcpEventPublisher.addStreamDefinition(streamDefinition); Thread.sleep(1000); log.info("Starting event client to send events to localhost:7612"); for (int i = 0; i < eventsToSend; i++) { tcpEventPublisher.sendEvent( streamDefinition.getId(), System.currentTimeMillis(), dataProvider.getEvent(), true); if (delay > 0) { Thread.sleep(delay); } } } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } finally { if (tcpEventPublisher != null) { tcpEventPublisher.shutdown(); } } }
public List<StreamDefinition> getSiddhiStreams( String[] inputStreamDefinitions, String queryExpressions) throws SiddhiParserException { SiddhiManager siddhiManager = createMockSiddhiManager(inputStreamDefinitions, queryExpressions); List<org.wso2.siddhi.query.api.definition.StreamDefinition> streamDefinitions = siddhiManager.getStreamDefinitions(); List<StreamDefinition> databridgeStreamDefinitions = new ArrayList<StreamDefinition>(streamDefinitions.size()); for (org.wso2.siddhi.query.api.definition.StreamDefinition siddhiStreamDef : streamDefinitions) { StreamConfiguration streamConfig = new StreamConfiguration(siddhiStreamDef.getStreamId()); StreamDefinition databridgeStreamDef = EventProcessorUtil.convertToDatabridgeStreamDefinition(siddhiStreamDef, streamConfig); databridgeStreamDefinitions.add(databridgeStreamDef); } siddhiManager.shutdown(); return databridgeStreamDefinitions; }
@Test public void testCreatingStreamDefinition() { StreamDefinition streamDefinition = SiddhiCompiler.parseStreamDefinition( "define stream StockStream ( symbol string, price int, volume float );"); StreamDefinition api = StreamDefinition.id("StockStream") .attribute("symbol", Attribute.Type.STRING) .attribute("price", Attribute.Type.INT) .attribute("volume", Attribute.Type.FLOAT); Assert.assertEquals(api, streamDefinition); }
private static void populateAttributes( org.wso2.siddhi.query.api.definition.StreamDefinition streamDefinition, List<Attribute> attributes, String prefix) { if (attributes != null) { for (Attribute attribute : attributes) { org.wso2.siddhi.query.api.definition.Attribute siddhiAttribute = EventProcessorUtil.convertToSiddhiAttribute(attribute, prefix); streamDefinition.attribute(siddhiAttribute.getName(), siddhiAttribute.getType()); } } }
public List<StratioStreamingMessage> list() { List<StratioStreamingMessage> result = new ArrayList<>(); for (StreamDefinition streamDefinition : siddhiManager.getStreamDefinitions()) { if (suitableToList(streamDefinition.getStreamId())) { StratioStreamingMessage message = new StratioStreamingMessage(); for (Attribute attribute : streamDefinition.getAttributeList()) { message.addColumn( new ColumnNameTypeValue( attribute.getName(), this.getStreamingType(attribute.getType()), null)); } StreamStatusDTO streamStatus = streamStatusDao.get(streamDefinition.getStreamId()); if (streamStatus != null) { Map<String, QueryDTO> attachedQueries = streamStatus.getAddedQueries(); for (Map.Entry<String, QueryDTO> entry : attachedQueries.entrySet()) { message.addQuery(new StreamQuery(entry.getKey(), entry.getValue().getQueryRaw())); } message.setUserDefined(streamStatus.getUserDefined()); message.setActiveActions( streamStatusDao.getEnabledActions(streamDefinition.getStreamId())); } message.setStreamName(streamDefinition.getStreamId()); result.add(message); } } return result; }
@Test public void testEqualObjects() throws SiddhiParserException { StreamDefinition streamDefinition = SiddhiCompiler.parseStreamDefinition( "@Foo(name='bar','Custom')define stream cseStream ( symbol string, price int, volume float )"); Assert.assertEquals( StreamDefinition.id("cseStream") .attribute("symbol", Attribute.Type.STRING) .attribute("price", Attribute.Type.INT) .attribute("volume", Attribute.Type.FLOAT) .annotation(Annotation.annotation("Foo").element("name", "bar").element("Custom")), streamDefinition); }
@Test public void testCreatingStreamDefinition2() { StreamDefinition streamDefinition = SiddhiCompiler.parseStreamDefinition( "define stream StockStream ( symbol string, price int, volume double, data Object );"); StreamDefinition api = StreamDefinition.id("StockStream") .attribute("symbol", Attribute.Type.STRING) .attribute("price", Attribute.Type.INT) .attribute("volume", Attribute.Type.DOUBLE) .attribute("data", Attribute.Type.OBJECT); Assert.assertEquals(api, streamDefinition); }
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { if (siddhiManager == null) { init(); } // Declaring output fileds for each exported stream ID for (String streamId : exportedStreamIds) { StreamDefinition streamDefinition = siddhiManager.getStreamDefinition(streamId); if (streamDefinition == null) { throw new RuntimeException("Cannot find exported stream - " + streamId); } List<String> list = new ArrayList<String>(); for (Attribute attribute : streamDefinition.getAttributeList()) { list.add(attribute.getName()); } Fields fields = new Fields(list); declarer.declareStream(streamId, fields); log.info("Declaring output field for stream -" + streamId); } }
public void addStreamDefinition(StreamDefinition streamDefinition) { String streamId = streamDefinition.getId(); this.streamRuntimeInfoMap.put( streamId, EventServerUtils.createStreamRuntimeInfo(streamDefinition)); }
public void addExecutionPlanConfiguration( ExecutionPlanConfiguration executionPlanConfiguration, AxisConfiguration axisConfiguration) throws ExecutionPlanDependencyValidationException, ExecutionPlanConfigurationException, ServiceDependencyValidationException { int tenantId = PrivilegedCarbonContext.getThreadLocalCarbonContext().getTenantId(); Map<String, ExecutionPlan> tenantExecutionPlans = tenantSpecificExecutionPlans.get(tenantId); if (tenantExecutionPlans == null) { tenantExecutionPlans = new ConcurrentHashMap<String, ExecutionPlan>(); tenantSpecificExecutionPlans.put(tenantId, tenantExecutionPlans); } else if (tenantExecutionPlans.get(executionPlanConfiguration.getName()) != null) { // if an execution plan with the same name already exists, we are not going to override it // with this plan. throw new ExecutionPlanConfigurationException( "Execution plan with the same name already exists. Please remove it and retry."); } // This iteration exists only as a check. Actual usage of imported stream configs is further // down for (StreamConfiguration streamConfiguration : executionPlanConfiguration.getImportedStreams()) { try { StreamDefinition streamDefinition = EventProcessorValueHolder.getEventStreamService() .getStreamDefinition(streamConfiguration.getStreamId(), tenantId); if (streamDefinition == null) { throw new ExecutionPlanDependencyValidationException( streamConfiguration.getStreamId(), "Imported Stream " + streamConfiguration.getStreamId() + " does not exist"); } } catch (EventStreamConfigurationException e) { throw new ExecutionPlanConfigurationException( "Error in retrieving stream ID : " + streamConfiguration.getStreamId()); } } // This iteration exists only as a check. Actual usage of exported stream configs is further // down for (StreamConfiguration streamConfiguration : executionPlanConfiguration.getExportedStreams()) { try { StreamDefinition streamDefinition = EventProcessorValueHolder.getEventStreamService() .getStreamDefinition(streamConfiguration.getStreamId(), tenantId); if (streamDefinition == null) { throw new ExecutionPlanDependencyValidationException( streamConfiguration.getStreamId(), "Exported Stream " + streamConfiguration.getStreamId() + " does not exist"); } } catch (EventStreamConfigurationException e) { throw new ExecutionPlanConfigurationException( "Error in retrieving stream ID : " + streamConfiguration.getStreamId()); } } Map<String, InputHandler> inputHandlerMap = new ConcurrentHashMap<String, InputHandler>( executionPlanConfiguration.getImportedStreams().size()); SiddhiConfiguration siddhiConfig = getSiddhiConfigurationFor(executionPlanConfiguration, tenantId); SiddhiManager siddhiManager = getSiddhiManagerFor(executionPlanConfiguration, siddhiConfig, inputHandlerMap); for (StreamConfiguration importedStreamConfiguration : executionPlanConfiguration.getImportedStreams()) { org.wso2.siddhi.query.api.definition.StreamDefinition siddhiStreamDefinition = new org.wso2.siddhi.query.api.definition.StreamDefinition(); siddhiStreamDefinition.name(importedStreamConfiguration.getSiddhiStreamName()); StreamDefinition streamDefinition = null; try { streamDefinition = EventProcessorValueHolder.getEventStreamService() .getStreamDefinition(importedStreamConfiguration.getStreamId(), tenantId); populateAttributes( siddhiStreamDefinition, streamDefinition.getMetaData(), EventProcessorConstants.META + EventProcessorConstants.ATTRIBUTE_SEPARATOR); populateAttributes( siddhiStreamDefinition, streamDefinition.getCorrelationData(), EventProcessorConstants.CORRELATION + EventProcessorConstants.ATTRIBUTE_SEPARATOR); populateAttributes(siddhiStreamDefinition, streamDefinition.getPayloadData(), ""); InputHandler inputHandler = siddhiManager.defineStream(siddhiStreamDefinition); inputHandlerMap.put(streamDefinition.getStreamId(), inputHandler); log.debug("input handler created for " + siddhiStreamDefinition.getStreamId()); } catch (EventStreamConfigurationException e) { // ignored as this will not happen } } for (StreamConfiguration exportedStreamConfiguration : executionPlanConfiguration.getExportedStreams()) { org.wso2.siddhi.query.api.definition.StreamDefinition siddhiStreamDefinition = new org.wso2.siddhi.query.api.definition.StreamDefinition(); siddhiStreamDefinition.name(exportedStreamConfiguration.getSiddhiStreamName()); StreamDefinition streamDefinition = null; try { streamDefinition = EventProcessorValueHolder.getEventStreamService() .getStreamDefinition(exportedStreamConfiguration.getStreamId(), tenantId); populateAttributes( siddhiStreamDefinition, streamDefinition.getMetaData(), EventProcessorConstants.META + EventProcessorConstants.ATTRIBUTE_SEPARATOR); populateAttributes( siddhiStreamDefinition, streamDefinition.getCorrelationData(), EventProcessorConstants.CORRELATION + EventProcessorConstants.ATTRIBUTE_SEPARATOR); populateAttributes(siddhiStreamDefinition, streamDefinition.getPayloadData(), ""); siddhiManager.defineStream(siddhiStreamDefinition); log.debug("stream defined for " + siddhiStreamDefinition.getStreamId()); } catch (EventStreamConfigurationException e) { // ignored as this will not happen } } HAManager haManager = null; String isDistributedProcessingEnabledString = executionPlanConfiguration .getSiddhiConfigurationProperties() .get(EventProcessorConstants.SIDDHI_DISTRIBUTED_PROCESSING); if (isDistributedProcessingEnabledString != null && isDistributedProcessingEnabledString.equalsIgnoreCase("RedundantNode")) { haManager = new HAManager( EventProcessorValueHolder.getHazelcastInstance(), executionPlanConfiguration.getName(), tenantId, siddhiManager, inputHandlerMap.size(), currentCepMembershipInfo); } try { siddhiManager.addExecutionPlan(executionPlanConfiguration.getQueryExpressions()); } catch (Exception e) { throw new ExecutionPlanConfigurationException( "Invalid query specified, " + e.getMessage(), e); } ExecutionPlan executionPlan = new ExecutionPlan( executionPlanConfiguration.getName(), siddhiManager, executionPlanConfiguration, haManager); tenantExecutionPlans.put(executionPlanConfiguration.getName(), executionPlan); // subscribe output to junction SindhiStormOutputEventListener stormOutputListener = null; if (isRunningOnStorm) { stormOutputListener = new SindhiStormOutputEventListener(executionPlanConfiguration, tenantId); } for (StreamConfiguration exportedStreamConfiguration : executionPlanConfiguration.getExportedStreams()) { SiddhiOutputStreamListener streamCallback; if (haManager != null) { streamCallback = new SiddhiHAOutputStreamListener( exportedStreamConfiguration.getSiddhiStreamName(), exportedStreamConfiguration.getStreamId(), executionPlanConfiguration, tenantId); haManager.addStreamCallback((SiddhiHAOutputStreamListener) streamCallback); } else { streamCallback = new SiddhiOutputStreamListener( exportedStreamConfiguration.getSiddhiStreamName(), exportedStreamConfiguration.getStreamId(), executionPlanConfiguration, tenantId); if (isRunningOnStorm) { stormOutputListener.registerOutputStreamListener( exportedStreamConfiguration.getSiddhiStreamName(), streamCallback); } } siddhiManager.addCallback(exportedStreamConfiguration.getSiddhiStreamName(), streamCallback); try { EventProcessorValueHolder.getEventStreamService().subscribe(streamCallback, tenantId); } catch (EventStreamConfigurationException e) { // ignored as this will never happen } executionPlan.addProducer(streamCallback); } // subscribe input to junction for (StreamConfiguration importedStreamConfiguration : executionPlanConfiguration.getImportedStreams()) { InputHandler inputHandler = inputHandlerMap.get(importedStreamConfiguration.getStreamId()); AbstractSiddhiInputEventDispatcher eventDispatcher = null; if (haManager != null) { eventDispatcher = new SiddhiHAInputEventDispatcher( importedStreamConfiguration.getStreamId(), inputHandler, executionPlanConfiguration, tenantId, haManager.getProcessThreadPoolExecutor(), haManager.getThreadBarrier()); haManager.addInputEventDispatcher( importedStreamConfiguration.getStreamId(), (SiddhiHAInputEventDispatcher) eventDispatcher); } else if (isRunningOnStorm) { StreamDefinition streamDefinition = null; try { streamDefinition = EventProcessorValueHolder.getEventStreamService() .getStreamDefinition(importedStreamConfiguration.getStreamId(), tenantId); } catch (EventStreamConfigurationException e) { // Ignore as this would never happen } eventDispatcher = new SiddhiStormInputEventDispatcher( streamDefinition, importedStreamConfiguration.getSiddhiStreamName(), executionPlanConfiguration, tenantId); } else { eventDispatcher = new SiddhiInputEventDispatcher( importedStreamConfiguration.getStreamId(), inputHandler, executionPlanConfiguration, tenantId); } try { EventProcessorValueHolder.getEventStreamService().subscribe(eventDispatcher, tenantId); executionPlan.addConsumer(eventDispatcher); } catch (EventStreamConfigurationException e) { // ignored as this will never happen } } if (haManager != null) { haManager.init(); } }
private static boolean validateSiddhiStreamWithDatabridgeStream( String streamName, String streamVersion, org.wso2.siddhi.query.api.definition.StreamDefinition siddhiStreamDefinition) throws ExecutionPlanConfigurationException, ExecutionPlanDependencyValidationException { if (siddhiStreamDefinition == null) { throw new ExecutionPlanDependencyValidationException( streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion, "Cannot validate null Siddhi stream for the stream: " + streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion + " "); } EventStreamService eventStreamService = EventProcessorValueHolder.getEventStreamService(); try { StreamDefinition streamDefinition = eventStreamService.getStreamDefinition(streamName, streamVersion); if (streamDefinition != null) { String siddhiAttributeName; int attributeCount = 0; int streamSize = (streamDefinition.getMetaData() == null ? 0 : streamDefinition.getMetaData().size()) + (streamDefinition.getCorrelationData() == null ? 0 : streamDefinition.getCorrelationData().size()) + (streamDefinition.getPayloadData() == null ? 0 : streamDefinition.getPayloadData().size()); if (siddhiStreamDefinition.getAttributeList().size() != streamSize) { throw new ExecutionPlanDependencyValidationException( streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion, "No of attributes in stream " + streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion + " do not match the no of attributes in Siddhi stream"); } if (streamDefinition.getMetaData() != null) { for (Attribute attribute : streamDefinition.getMetaData()) { siddhiAttributeName = EventProcessorConstants.META_PREFIX + attribute.getName(); org.wso2.siddhi.query.api.definition.Attribute.Type type = siddhiStreamDefinition.getAttributeType(siddhiAttributeName); // null check for type not required since an exception is thrown by Siddhi // StreamDefinition.getAttributeType() method for non-existent attributes if (siddhiStreamDefinition.getAttributePosition(siddhiAttributeName) != attributeCount++) { throw new ExecutionPlanDependencyValidationException( streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion, "Stream " + streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion + "; Attribute positions do not match for attribute: " + attribute.getName()); } if (!isMatchingType(type, attribute.getType())) { throw new ExecutionPlanDependencyValidationException( streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion, "Stream " + streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion + "; Type mismatch for attribute: " + attribute.getName()); } } } if (streamDefinition.getCorrelationData() != null) { for (Attribute attribute : streamDefinition.getCorrelationData()) { siddhiAttributeName = EventProcessorConstants.CORRELATION_PREFIX + attribute.getName(); org.wso2.siddhi.query.api.definition.Attribute.Type type = siddhiStreamDefinition.getAttributeType(siddhiAttributeName); // null check for type not required since an exception is thrown by Siddhi // StreamDefinition.getAttributeType() method for non-existent attributes if (siddhiStreamDefinition.getAttributePosition(siddhiAttributeName) != attributeCount++) { throw new ExecutionPlanDependencyValidationException( streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion, "Stream " + streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion + "; Attribute positions do not match for attribute: " + attribute.getName()); } if (!isMatchingType(type, attribute.getType())) { throw new ExecutionPlanDependencyValidationException( streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion, "Stream " + streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion + "; Type mismatch for attribute: " + attribute.getName()); } } } if (streamDefinition.getPayloadData() != null) { for (Attribute attribute : streamDefinition.getPayloadData()) { siddhiAttributeName = attribute.getName(); org.wso2.siddhi.query.api.definition.Attribute.Type type = siddhiStreamDefinition.getAttributeType(siddhiAttributeName); // null check for type not required since an exception is thrown by Siddhi // StreamDefinition.getAttributeType() method for non-existent attributes if (siddhiStreamDefinition.getAttributePosition(siddhiAttributeName) != attributeCount++) { throw new ExecutionPlanDependencyValidationException( streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion, "Stream " + streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion + "; Attribute positions do not match for attribute: " + attribute.getName()); } if (!isMatchingType(type, attribute.getType())) { throw new ExecutionPlanDependencyValidationException( streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion, "Stream " + streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion + "; Type mismatch for attribute: " + attribute.getName()); } } } return true; } } catch (EventStreamConfigurationException e) { throw new ExecutionPlanConfigurationException( "Error while validating stream definition with store : " + e.getMessage(), e); } catch (AttributeNotExistException e) { throw new ExecutionPlanDependencyValidationException( streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion, e.getMessage()); } throw new ExecutionPlanDependencyValidationException( streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion, "Stream " + streamName + EventProcessorConstants.STREAM_SEPARATOR + streamVersion + " does not exist"); }