@RequestMapping(value = "/data/dateranges", method = RequestMethod.GET)
public ResponseEntity<String> getDateRange(
@RequestParam(value = PARAM_COLLECTION, required = true) String collection)
throws IOException {
CollectionSchemaInfo info = collectionsConfig.getSchema(collection);
ReportData data = collectionsConfig.getReportData(collection);
StringWriter writer = new StringWriter();
JsonGenerator g = searchService.writeSearchResponseStart(writer, null);
if (data != null) {
for (String dateField : data.getDateFields()) {
List<Date> dates = searchService.getSolrFieldDateRange(collection, dateField, info);
if (dates == null || dates.size() != 2) {
continue;
}
g.writeObjectFieldStart(dateField);
g.writeStringField(DATE_RANGE_START_KEY, dates.get(0).toString());
g.writeStringField(DATE_RANGE_END_KEY, dates.get(1).toString());
g.writeEndObject();
}
}
searchService.writeSearchResponseEnd(g);
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.put(CONTENT_TYPE_HEADER, singletonList(CONTENT_TYPE_VALUE));
return new ResponseEntity<String>(writer.toString(), httpHeaders, OK);
}
public void toJson(JsonGenerator gen) throws IOException {
gen.writeStartObject();
if (type != null) {
gen.writeStringField("type", type.getName());
}
if (tableName != null) {
gen.writeStringField("tableName", tableName);
}
if (versionUpdated != -1) {
gen.writeNumberField("versionUpdated", versionUpdated);
}
if (versionCreated != -1) {
gen.writeNumberField("versionCreated", versionCreated);
}
if (recordTypeChanged) {
gen.writeBooleanField("recordTypeChanged", true);
}
if (updatedFields != null && updatedFields.size() > 0) {
gen.writeArrayFieldStart("updatedFields");
for (SchemaId updatedField : updatedFields) {
gen.writeBinary(updatedField.getBytes());
}
gen.writeEndArray();
}
if (vtagsToIndex != null && vtagsToIndex.size() > 0) {
gen.writeArrayFieldStart("vtagsToIndex");
for (SchemaId vtag : vtagsToIndex) {
gen.writeBinary(vtag.getBytes());
}
gen.writeEndArray();
}
if (attributes != null && attributes.size() > 0) {
gen.writeObjectFieldStart("attributes");
for (String key : attributes.keySet()) {
gen.writeStringField(key, attributes.get(key));
}
gen.writeEndObject();
}
if (indexRecordFilterData != null) {
gen.writeFieldName("indexFilterData");
indexRecordFilterData.toJson(gen);
}
gen.writeEndObject();
gen.flush();
}
void toJson(JsonGenerator gen) throws IOException {
gen.writeStartObject();
gen.writeArrayFieldStart("sourceSchemas");
for (Schema schema : schemas) {
schema.toJson(gen);
}
gen.writeEndArray();
gen.writeObjectFieldStart("fieldAliases");
for (Map.Entry<String, Map<String, String>> entry : schemaFieldAliases.entrySet()) {
String schemaName = entry.getKey();
Map<String, String> aliases = entry.getValue();
gen.writeObjectField(schemaName, aliases);
}
gen.writeEndObject();
gen.writeArrayFieldStart("groupByFields");
for (String field : groupByFields) {
gen.writeString(field);
}
gen.writeEndArray();
if (customPartitionFields != null && !customPartitionFields.isEmpty()) {
gen.writeArrayFieldStart("customPartitionFields");
for (String field : customPartitionFields) {
gen.writeString(field);
}
gen.writeEndArray();
}
if (rollupFrom != null) {
gen.writeFieldName("rollupFrom");
gen.writeString(rollupFrom);
}
gen.writeFieldName("commonOrderBy");
commonCriteria.toJson(gen);
gen.writeStringField("schemasOrder", schemasOrder.toString());
// TODO this code should write a map with sourceName
if (specificCriterias == null || specificCriterias.isEmpty()) {
initSecondaryCriteriasWithNull();
}
gen.writeArrayFieldStart("specificOrderBys");
for (Criteria c : specificCriterias) {
if (c == null) {
gen.writeNull();
} else {
c.toJson(gen);
}
}
gen.writeEndArray();
gen.writeEndObject();
}
protected void writeExtendedInfos(JsonGenerator jg, LogEntry logEntry) throws IOException {
Map<String, ExtendedInfo> extended = logEntry.getExtendedInfos();
jg.writeObjectFieldStart("extended");
for (String key : extended.keySet()) {
ExtendedInfo ei = extended.get(key);
if (ei != null && ei.getSerializableValue() != null) {
writeExtendedInfo(jg, key, ei.getSerializableValue());
} else {
jg.writeNullField(key);
}
}
jg.writeEndObject();
}
@Override
public void run() {
try {
// command metrics
for (HystrixCommandMetrics commandMetrics : HystrixCommandMetrics.getInstances()) {
HystrixCommandKey key = commandMetrics.getCommandKey();
HystrixCircuitBreaker circuitBreaker = HystrixCircuitBreaker.Factory.getInstance(key);
StringWriter jsonString = new StringWriter();
JsonGenerator json = jsonFactory.createJsonGenerator(jsonString);
// Informational and Status
json.writeStartObject();
json.writeStringField(type.value, HystrixCommand.value);
json.writeStringField(name.value, key.name());
json.writeStringField(group.value, commandMetrics.getCommandGroup().name());
json.writeNumberField(currentTime.value, (int) (System.currentTimeMillis() / 1000));
// circuit breaker
json.writeBooleanField(isCircuitBreakerOpen.value, circuitBreaker.isOpen());
HystrixCommandMetrics.HealthCounts healthCounts = commandMetrics.getHealthCounts();
json.writeNumberField(errorPercentage.value, healthCounts.getErrorPercentage());
json.writeNumberField(errorCount.value, healthCounts.getErrorCount());
json.writeNumberField(requestCount.value, healthCounts.getTotalRequests());
// rolling counters Gauge
json.writeNumberField(
rollingCountCollapsedRequests.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.COLLAPSED));
json.writeNumberField(
rollingCountExceptionsThrown.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN));
json.writeNumberField(
rollingCountFailure.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FAILURE));
json.writeNumberField(
rollingCountFallbackFailure.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE));
json.writeNumberField(
rollingCountFallbackRejection.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION));
json.writeNumberField(
rollingCountFallbackSuccess.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS));
json.writeNumberField(
rollingCountResponsesFromCache.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE));
json.writeNumberField(
rollingCountSemaphoreRejected.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED));
json.writeNumberField(
rollingCountShortCircuited.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED));
json.writeNumberField(
rollingCountSuccess.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS));
json.writeNumberField(
rollingCountThreadPoolRejected.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED));
json.writeNumberField(
rollingCountTimeout.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT));
json.writeNumberField(
currentConcurrentExecutionCount.value,
commandMetrics.getCurrentConcurrentExecutionCount());
// latency percentiles
json.writeNumberField(latencyExecute_mean.value, commandMetrics.getExecutionTimeMean());
json.writeObjectFieldStart(latencyExecute.value);
json.writeNumberField("0", commandMetrics.getExecutionTimePercentile(0));
json.writeNumberField("25", commandMetrics.getExecutionTimePercentile(25));
json.writeNumberField("50", commandMetrics.getExecutionTimePercentile(50));
json.writeNumberField("75", commandMetrics.getExecutionTimePercentile(75));
json.writeNumberField("90", commandMetrics.getExecutionTimePercentile(90));
json.writeNumberField("95", commandMetrics.getExecutionTimePercentile(95));
json.writeNumberField("99", commandMetrics.getExecutionTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.getExecutionTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.getExecutionTimePercentile(100));
json.writeEndObject();
//
json.writeNumberField(latencyTotal_mean.value, commandMetrics.getTotalTimeMean());
json.writeObjectFieldStart(latencyTotal.value);
json.writeNumberField("0", commandMetrics.getTotalTimePercentile(0));
json.writeNumberField("25", commandMetrics.getTotalTimePercentile(25));
json.writeNumberField("50", commandMetrics.getTotalTimePercentile(50));
json.writeNumberField("75", commandMetrics.getTotalTimePercentile(75));
json.writeNumberField("90", commandMetrics.getTotalTimePercentile(90));
json.writeNumberField("95", commandMetrics.getTotalTimePercentile(95));
json.writeNumberField("99", commandMetrics.getTotalTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.getTotalTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.getTotalTimePercentile(100));
json.writeEndObject();
// property values for reporting what is actually seen by the command rather than what was
// set somewhere
HystrixCommandProperties commandProperties = commandMetrics.getProperties();
json.writeNumberField(
propertyValue_circuitBreakerRequestVolumeThreshold.value,
commandProperties.circuitBreakerRequestVolumeThreshold().get());
json.writeNumberField(
propertyValue_circuitBreakerSleepWindowInMilliseconds.value,
commandProperties.circuitBreakerSleepWindowInMilliseconds().get());
json.writeNumberField(
propertyValue_circuitBreakerErrorThresholdPercentage.value,
commandProperties.circuitBreakerErrorThresholdPercentage().get());
json.writeBooleanField(
propertyValue_circuitBreakerForceOpen.value,
commandProperties.circuitBreakerForceOpen().get());
json.writeBooleanField(
propertyValue_circuitBreakerForceClosed.value,
commandProperties.circuitBreakerForceClosed().get());
json.writeBooleanField(
propertyValue_circuitBreakerEnabled.value,
commandProperties.circuitBreakerEnabled().get());
json.writeStringField(
propertyValue_executionIsolationStrategy.value,
commandProperties.executionIsolationStrategy().get().name());
json.writeNumberField(
propertyValue_executionIsolationThreadTimeoutInMilliseconds.value,
commandProperties.executionIsolationThreadTimeoutInMilliseconds().get());
json.writeBooleanField(
propertyValue_executionIsolationThreadInterruptOnTimeout.value,
commandProperties.executionIsolationThreadInterruptOnTimeout().get());
json.writeStringField(
propertyValue_executionIsolationThreadPoolKeyOverride.value,
commandProperties.executionIsolationThreadPoolKeyOverride().get());
json.writeNumberField(
propertyValue_executionIsolationSemaphoreMaxConcurrentRequests.value,
commandProperties.executionIsolationSemaphoreMaxConcurrentRequests().get());
json.writeNumberField(
propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests.value,
commandProperties.fallbackIsolationSemaphoreMaxConcurrentRequests().get());
/*
* The following are commented out as these rarely change and are verbose for streaming for something people don't change.
* We could perhaps allow a property or request argument to include these.
*/
// json.put("propertyValue_metricsRollingPercentileEnabled",
// commandProperties.metricsRollingPercentileEnabled().get());
// json.put("propertyValue_metricsRollingPercentileBucketSize",
// commandProperties.metricsRollingPercentileBucketSize().get());
// json.put("propertyValue_metricsRollingPercentileWindow",
// commandProperties.metricsRollingPercentileWindowInMilliseconds().get());
// json.put("propertyValue_metricsRollingPercentileWindowBuckets",
// commandProperties.metricsRollingPercentileWindowBuckets().get());
// json.put("propertyValue_metricsRollingStatisticalWindowBuckets",
// commandProperties.metricsRollingStatisticalWindowBuckets().get());
json.writeNumberField(
propertyValue_metricsRollingStatisticalWindowInMilliseconds.value,
commandProperties.metricsRollingStatisticalWindowInMilliseconds().get());
json.writeBooleanField(
propertyValue_requestCacheEnabled.value, commandProperties.requestCacheEnabled().get());
json.writeBooleanField(
propertyValue_requestLogEnabled.value, commandProperties.requestLogEnabled().get());
json.writeNumberField(reportingHosts.value, 1);
json.writeStringField(Key.ip.value, app_ip);
json.writeEndObject();
json.close();
// System.out.println(ip + ":" + port + "||" +
// jsonString.getBuffer().toString());
UDPClient.send(
socketIp, socketPort, jsonString.getBuffer().toString().getBytes(), new byte[] {});
}
// thread pool metrics
for (HystrixThreadPoolMetrics threadPoolMetrics : HystrixThreadPoolMetrics.getInstances()) {
HystrixThreadPoolKey key = threadPoolMetrics.getThreadPoolKey();
StringWriter jsonString = new StringWriter();
JsonGenerator json = jsonFactory.createJsonGenerator(jsonString);
json.writeStartObject();
json.writeStringField(type.value, HystrixThreadPool.value);
json.writeStringField(name.value, key.name());
json.writeNumberField(currentTime.value, System.currentTimeMillis());
// 101.3 80 154
json.writeNumberField(
currentActiveCount.value, threadPoolMetrics.getCurrentActiveCount().intValue());
json.writeNumberField(
currentCompletedTaskCount.value,
threadPoolMetrics.getCurrentCompletedTaskCount().longValue());
json.writeNumberField(
currentCorePoolSize.value, threadPoolMetrics.getCurrentCorePoolSize().intValue());
json.writeNumberField(
currentLargestPoolSize.value, threadPoolMetrics.getCurrentLargestPoolSize().intValue());
json.writeNumberField(
currentMaximumPoolSize.value, threadPoolMetrics.getCurrentMaximumPoolSize().intValue());
json.writeNumberField(
currentPoolSize.value, threadPoolMetrics.getCurrentPoolSize().intValue());
json.writeNumberField(
currentQueueSize.value, threadPoolMetrics.getCurrentQueueSize().intValue());
json.writeNumberField(
currentTaskCount.value, threadPoolMetrics.getCurrentTaskCount().longValue());
json.writeNumberField(
rollingCountThreadsExecuted.value, threadPoolMetrics.getRollingCountThreadsExecuted());
json.writeNumberField(
rollingMaxActiveThreads.value, threadPoolMetrics.getRollingMaxActiveThreads());
json.writeNumberField(
propertyValue_queueSizeRejectionThreshold.value,
threadPoolMetrics.getProperties().queueSizeRejectionThreshold().get());
json.writeNumberField(
propertyValue_metricsRollingStatisticalWindowInMilliseconds.value,
threadPoolMetrics
.getProperties()
.metricsRollingStatisticalWindowInMilliseconds()
.get());
json.writeNumberField(
reportingHosts.value, 1); // this will get summed across all instances in a cluster
json.writeStringField(Key.ip.value, app_ip);
json.writeEndObject();
json.close();
String str = jsonString.getBuffer().toString();
byte[] ret = str.getBytes();
UDPClient.send(socketIp, socketPort, ret, new byte[] {});
}
} catch (Exception e) {
System.err.println("Failed to output metrics as JSON");
e.printStackTrace();
}
}
public static void writeNeuronCoreTop(
EDSimulation sim,
StringBuilder sb,
JsonGenerator g,
Map<String, Float> initialState,
String neuronName)
throws JsonGenerationException, IOException {
currentBit = 511; // 5119;
currentLength = 0;
Integer = 0;
sb.append(
"\r\n"
+ "library ieee;\r\n"
+ "use ieee.std_logic_1164.all;\r\n"
+ "use ieee.std_logic_unsigned.all;\r\n"
+ "library ieee_proposed;\r\n"
+ "use ieee_proposed.fixed_pkg.all;\r\n"
+ "use ieee_proposed.fixed_float_types.ALL;\r\n"
+ "use std.textio.all;\r\n"
+ "use ieee.std_logic_textio.all; -- if you're saving this type of signal\r\n"
+ "use IEEE.numeric_std.all;\r\n"
+ "");
sb.append(
"\r\n"
+ "entity neuroncore_top is\r\n"
+ " Port ( clk : in STD_LOGIC; --SYSTEM CLOCK, THIS ITSELF DOES NOT SIGNIFY TIME STEPS - AKA A SINGLE TIMESTEP MAY TAKE MANY CLOCK CYCLES\r\n"
+ " init_model : in STD_LOGIC; --SYNCHRONOUS RESET\r\n"
+ " step_once_go : in STD_LOGIC; --signals to the neuron from the core that a time step is to be simulated\r\n"
+ " step_once_complete : out STD_LOGIC; --signals to the core that a time step has finished\r\n"
+ " \r\n"
+ " mega_bus_in : in STD_LOGIC_Vector (99999 downto 0 );\r\n"
+ " mega_bus_out : out STD_LOGIC_Vector (99999 downto 0 )\r\n"
+ " );\r\n"
+ "end neuroncore_top;\r\n"
+ "\r\n"
+ "---------------------------------------------------------------------\r\n"
+ "\r\n"
+ "architecture top of neuroncore_top is\r\n");
EDComponent neuron = sim.neuronComponents.get(0);
for (int i = 0; i < sim.neuronComponents.size(); i++) {
if (sim.neuronComponents.get(i).name.matches(neuronName)) {
neuron = sim.neuronComponents.get(i);
break;
}
}
sb.append(
"\r\n"
+ "component top_synth\r\n"
+ " Port ( clk : in STD_LOGIC; --SYSTEM CLOCK, THIS ITSELF DOES NOT SIGNIFY TIME STEPS - AKA A SINGLE TIMESTEP MAY TAKE MANY CLOCK CYCLES\r\n"
+ " init_model : in STD_LOGIC; --SYNCHRONOUS RESET\r\n"
+ " step_once_go : in STD_LOGIC; --signals to the neuron from the core that a time step is to be simulated\r\n"
+ " step_once_complete : out STD_LOGIC; --signals to the core that a time step has finished\r\n"
+ " eventport_in_spike_aggregate : in STD_LOGIC_VECTOR(511 downto 0);\r\n"
+
// " SelectSpikesIn : in STD_LOGIC_VECTOR(4607 downto 0);\r\n" +
" ");
String name = "";
for (Iterator<EDEventPort> i = neuron.eventports.iterator(); i.hasNext(); ) {
EDEventPort item = i.next();
sb.append(
" "
+ neuron.name
+ "_eventport_"
+ item.direction
+ "_"
+ item.name
+ " : "
+ item.direction
+ " STD_LOGIC;\r\n");
}
if (neuron.regimes.size() > 0) {
sb.append(
" "
+ neuron.name
+ "current_regime_in_stdlv : in STD_LOGIC_VECTOR(1 downto 0);\r\n"
+ " "
+ neuron.name
+ "current_regime_out_stdlv : out STD_LOGIC_VECTOR(1 downto 0);\r\n");
}
TopSynth.writeEntitySignals(neuron, sb, name, neuron.name + "_");
sb.append(
"\r\n"
+ " sysparam_time_timestep : sfixed (-6 downto -22);\r\n"
+ " sysparam_time_simtime : sfixed (6 downto -22)\r\n"
+ " );\r\n"
+ "end component;\r\n"
+ "\r\n"
+ "\r\n"
+ " \r\n"
+ " ");
for (Iterator<EDEventPort> i = neuron.eventports.iterator(); i.hasNext(); ) {
EDEventPort item = i.next();
sb.append(
"signal "
+ neuron.name
+ "_eventport_"
+ item.direction
+ "_"
+ item.name
+ "_internal : std_logic;\r\n");
}
if (neuron.regimes.size() > 0) {
sb.append(
"signal "
+ neuron.name
+ "_current_regime_out_stdlv_int : STD_LOGIC_VECTOR(1 downto 0);\r\n");
}
writeStateSignals(neuron, sb, "", neuron.name);
g.writeObjectFieldStart(neuron.name);
g.writeObjectFieldStart("SpikesIn");
g.writeNumberField("bottom", 0);
g.writeNumberField("top", 511);
g.writeEndObject();
g.writeArrayFieldStart("NeuronVariablesIn");
sb.append(
"begin\r\n"
+ "\r\n"
+ "top_synth_uut : neuron_model \r\n"
+ " port map ( clk => clk,\r\n"
+ " init_model => init_model, \r\n"
+ " step_once_go => step_once_go,\r\n"
+ " step_once_complete => step_once_complete,\r\n"
+ " eventport_in_spike_aggregate => mega_bus_in(511 downto 0),\r\n"
+
// " SelectSpikesIn => mega_bus_in(5119 downto 512),\r\n" +
" ");
for (Iterator<EDEventPort> i = neuron.eventports.iterator(); i.hasNext(); ) {
EDEventPort item = i.next();
sb.append(
" "
+ neuron.name
+ "_eventport_"
+ item.direction
+ "_"
+ item.name
+ " => "
+ neuron.name
+ "_eventport_"
+ item.direction
+ "_"
+ item.name
+ "_internal ,\r\n");
}
if (neuron.regimes.size() > 0) {
currentBit = currentBit + currentLength + 1;
while (currentBit % 8 != 0) currentBit++;
currentLength = 1;
int currentTop = currentBit + currentLength;
writeSignal("regime", currentBit, currentTop, 0, 0, g);
sb.append(
""
+ neuron.name
+ "current_regime_in_stdlv => mega_bus_in("
+ currentTop
+ " downto "
+ currentBit
+ "),\r\n");
sb.append(
""
+ neuron.name
+ "current_regime_out_stdlv => "
+ neuron.name
+ "_current_regime_out_stdlv_int,\r\n");
}
writeConnectivityMapVar(neuron, sb, "", neuron.name, g);
g.writeEndArray();
g.writeArrayFieldStart("NeuronParameters");
writeConnectivityMapPar(neuron, sb, "", neuron.name, g);
currentBit = currentBit + currentLength + 1;
currentLength = 16;
int currentTop = currentBit + currentLength;
sb.append(
"sysparam_time_timestep => to_sfixed (mega_bus_in("
+ currentTop
+ " downto "
+ currentBit
+ "),-6 , -22),");
currentBit = currentBit + currentLength + 1;
currentLength = 28;
currentTop = currentBit + currentLength;
sb.append(
"sysparam_time_simtime => to_sfixed (mega_bus_in("
+ currentTop
+ " downto "
+ currentBit
+ "),6, -22)");
sb.append("\r\n" + " );\r\n");
g.writeEndArray();
g.writeArrayFieldStart("NeuronVariablesOut");
currentBit = -1;
currentLength = 0;
if (neuron.regimes.size() > 0) {
currentBit = currentBit + currentLength + 1;
while (currentBit % 8 != 0) currentBit++;
currentLength = 1;
currentTop = currentBit + currentLength;
writeSignal("regime", currentBit, currentTop, 0, 0, g);
sb.append(
" mega_bus_out("
+ currentTop
+ " downto "
+ currentBit
+ ") <= "
+ neuron.name
+ "_current_regime_out_stdlv_int;\r\n");
}
writeStateToBusSignals(neuron, sb, "", neuron.name, g);
for (Iterator<EDEventPort> i = neuron.eventports.iterator(); i.hasNext(); ) {
EDEventPort item = i.next();
currentBit = currentBit + currentLength + 1;
while (currentBit % 8 != 0) currentBit++;
sb.append(
" mega_bus_out("
+ currentBit
+ ") <= "
+ neuron.name
+ "_eventport_"
+ item.direction
+ "_"
+ item.name
+ "_internal;\r\n");
}
sb.append("\r\n" + "end top;\r\n" + "");
g.writeEndArray();
g.writeEndObject();
}