public MessageInput getPersisted(String inputId) {
for (MessageInput input : getAllPersisted()) {
if (input.getId().equals(inputId)) return input;
}
return null;
}
@GET
@Timed
@ApiOperation(value = "Get information of a single extractor of an input")
@Path("/{extractorId}")
@ApiResponses(
value = {
@ApiResponse(code = 404, message = "No such input on this node."),
@ApiResponse(code = 404, message = "No such extractor on this input.")
})
@Produces(MediaType.APPLICATION_JSON)
public ExtractorSummary single(
@ApiParam(name = "inputId", required = true) @PathParam("inputId") String inputId,
@ApiParam(name = "extractorId", required = true) @PathParam("extractorId")
final String extractorId)
throws NotFoundException {
checkPermission(RestPermissions.INPUTS_READ, inputId);
final MessageInput input = persistedInputs.get(inputId);
if (input == null) {
LOG.error("Input <{}> not found.", inputId);
throw new javax.ws.rs.NotFoundException("Couldn't find input " + inputId);
}
final Input mongoInput = inputService.find(input.getPersistId());
final Extractor extractor = inputService.getExtractor(mongoInput, extractorId);
return toSummary(extractor);
}
@Subscribe
public void inputCreated(InputCreated inputCreatedEvent) {
LOG.debug("Input created/changed: " + inputCreatedEvent.id());
final Input input;
try {
input = inputService.find(inputCreatedEvent.id());
} catch (NotFoundException e) {
LOG.warn("Received InputCreated event but could not find Input: ", e);
return;
}
final IOState<MessageInput> inputState = inputRegistry.getInputState(inputCreatedEvent.id());
if (inputState != null) {
inputRegistry.remove(inputState);
}
if (!input.isGlobal() && !this.nodeId.toString().equals(input.getNodeId())) {
return;
}
final MessageInput messageInput;
try {
messageInput = inputService.getMessageInput(input);
messageInput.initialize();
} catch (NoSuchInputTypeException e) {
LOG.warn("Newly created input is of invalid type: " + input.getType(), e);
return;
}
final IOState<MessageInput> newInputState = inputLauncher.launch(messageInput);
inputRegistry.add(newInputState);
}
@POST
@Timed
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Add an extractor to an input", response = ExtractorCreated.class)
@ApiResponses(
value = {
@ApiResponse(code = 404, message = "No such input on this node."),
@ApiResponse(code = 400, message = "No such extractor type."),
@ApiResponse(code = 400, message = "Field the extractor should write on is reserved."),
@ApiResponse(code = 400, message = "Missing or invalid configuration.")
})
public Response create(
@ApiParam(name = "inputId", required = true) @PathParam("inputId") String inputId,
@ApiParam(name = "JSON body", required = true) @Valid @NotNull CreateExtractorRequest cer)
throws NotFoundException {
checkPermission(RestPermissions.INPUTS_EDIT, inputId);
final MessageInput input = inputs.getRunningInput(inputId);
if (input == null) {
LOG.error("Input <{}> not found.", inputId);
throw new javax.ws.rs.NotFoundException();
}
final Input mongoInput = inputService.find(input.getPersistId());
final String id = new com.eaio.uuid.UUID().toString();
final Extractor extractor = buildExtractorFromRequest(cer, id);
try {
inputService.addExtractor(mongoInput, extractor);
} catch (ValidationException e) {
LOG.error("Extractor persist validation failed.", e);
throw new BadRequestException(e);
}
final String msg =
"Added extractor <"
+ id
+ "> of type ["
+ cer.extractorType()
+ "] to input <"
+ inputId
+ ">.";
LOG.info(msg);
activityWriter.write(new Activity(msg, ExtractorsResource.class));
final ExtractorCreated result = ExtractorCreated.create(id);
final URI extractorUri =
getUriBuilderToSelf().path(ExtractorsResource.class).path("{inputId}").build(input.getId());
return Response.created(extractorUri).entity(result).build();
}
@PUT
@Timed
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Update an extractor")
@Path("/{extractorId}")
@ApiResponses(
value = {
@ApiResponse(code = 404, message = "No such input on this node."),
@ApiResponse(code = 404, message = "No such extractor on this input."),
@ApiResponse(code = 400, message = "No such extractor type."),
@ApiResponse(code = 400, message = "Field the extractor should write on is reserved."),
@ApiResponse(code = 400, message = "Missing or invalid configuration.")
})
public ExtractorSummary update(
@ApiParam(name = "inputId", required = true) @PathParam("inputId") String inputId,
@ApiParam(name = "extractorId", required = true) @PathParam("extractorId") String extractorId,
@ApiParam(name = "JSON body", required = true) @Valid @NotNull CreateExtractorRequest cer)
throws NotFoundException {
checkPermission(RestPermissions.INPUTS_EDIT, inputId);
final MessageInput input = persistedInputs.get(inputId);
if (input == null) {
LOG.error("Input <{}> not found.", inputId);
throw new javax.ws.rs.NotFoundException("Couldn't find input " + inputId);
}
final Input mongoInput = inputService.find(input.getPersistId());
final Extractor originalExtractor = inputService.getExtractor(mongoInput, extractorId);
final Extractor extractor = buildExtractorFromRequest(cer, originalExtractor.getId());
inputService.removeExtractor(mongoInput, originalExtractor.getId());
try {
inputService.addExtractor(mongoInput, extractor);
} catch (ValidationException e) {
LOG.error("Extractor persist validation failed.", e);
throw new BadRequestException(e);
}
final String msg =
"Updated extractor <"
+ originalExtractor.getId()
+ "> of type ["
+ cer.extractorType()
+ "] in input <"
+ inputId
+ ">.";
LOG.info(msg);
activityWriter.write(new Activity(msg, ExtractorsResource.class));
return toSummary(extractor);
}
public InputState stop(MessageInput input) {
InputState inputState = getRunningInputState(input.getId());
if (inputState != null) {
try {
input.stop();
} catch (Exception e) {
LOG.warn("Stopping input <{}> failed, removing anyway: {}", input.getId(), e);
}
removeFromRunning(input);
inputState.setState(InputState.InputStateType.STOPPED);
finishedStop(inputState);
}
return inputState;
}
@Override
protected void shutDown() throws Exception {
LOG.debug("Stopping InputSetupService");
eventBus.unregister(this);
for (InputState state : inputRegistry.getRunningInputs()) {
MessageInput input = state.getMessageInput();
LOG.info(
"Attempting to close input <{}> [{}].", input.getUniqueReadableId(), input.getName());
Stopwatch s = Stopwatch.createStarted();
try {
input.stop();
LOG.info(
"Input <{}> closed. Took [{}ms]",
input.getUniqueReadableId(),
s.elapsed(TimeUnit.MILLISECONDS));
} catch (Exception e) {
LOG.error(
"Unable to stop input <{}> [{}]: " + e.getMessage(),
input.getUniqueReadableId(),
input.getName());
} finally {
s.stop();
}
}
LOG.debug("Stopped InputSetupService");
}
@Before
public void setUp() {
when(messageInput.getUniqueReadableId()).thenReturn("test");
metricRegistry = new MetricRegistry();
listener =
new ClientListener(
messageInput,
Collections.singletonList(new Subscription("test", QoS.AT_LEAST_ONCE)),
metricRegistry);
}
protected void handleLaunchException(Throwable e, MessageInput input, InputState inputState) {
StringBuilder msg =
new StringBuilder(
"The ["
+ input.getClass().getCanonicalName()
+ "] input with ID <"
+ input.getId()
+ "> misfired. Reason: ");
String causeMsg = extractMessageCause(e);
msg.append(causeMsg);
LOG.error(msg.toString(), e);
// Clean up.
// cleanInput(input);
inputState.setState(InputState.InputStateType.FAILED);
inputState.setDetailedMessage(causeMsg);
}
@DELETE
@Timed
@ApiOperation(value = "Delete an extractor")
@Path("/{extractorId}")
@ApiResponses(
value = {
@ApiResponse(code = 400, message = "Invalid request."),
@ApiResponse(code = 404, message = "Input not found."),
@ApiResponse(code = 404, message = "Extractor not found.")
})
@Produces(MediaType.APPLICATION_JSON)
public void terminate(
@ApiParam(name = "inputId", required = true) @PathParam("inputId") String inputId,
@ApiParam(name = "extractorId", required = true) @PathParam("extractorId") String extractorId)
throws NotFoundException {
checkPermission(RestPermissions.INPUTS_EDIT, inputId);
final MessageInput input = persistedInputs.get(inputId);
if (input == null) {
LOG.error("Input <{}> not found.", inputId);
throw new javax.ws.rs.NotFoundException("Couldn't find input " + inputId);
}
// Remove from Mongo.
final Input mongoInput = inputService.find(input.getPersistId());
final Extractor extractor = inputService.getExtractor(mongoInput, extractorId);
inputService.removeExtractor(mongoInput, extractor.getId());
final String msg =
"Deleted extractor <"
+ extractorId
+ "> of type ["
+ extractor.getType()
+ "] "
+ "from input <"
+ inputId
+ ">.";
LOG.info(msg);
activityWriter.write(new Activity(msg, InputsResource.class));
}
@Override
public void deliver(String shortMessage, String source, Map<String, Object> fields) {
try {
final Map<String, Object> data = Maps.newHashMap();
data.put("short_message", shortMessage);
data.put("host", source);
data.putAll(fields);
final byte[] payload = mapper.writeValueAsBytes(data);
input.processRawMessage(new RawMessage(payload));
} catch (JsonProcessingException e) {
log.error("Unable to serialized metrics", e);
}
}
@Override
protected void startCommand() {
LOG.info(
"Graylog " + commandName + " {} starting up. (JRE: {})",
version,
Tools.getSystemInformation());
// Do not use a PID file if the user requested not to
if (!isNoPidFile()) {
savePidFile(getPidFile());
}
final ServerStatus serverStatus = injector.getInstance(ServerStatus.class);
serverStatus.initialize();
startNodeRegistration(injector);
final ActivityWriter activityWriter;
final ServiceManager serviceManager;
try {
activityWriter = injector.getInstance(ActivityWriter.class);
serviceManager = injector.getInstance(ServiceManager.class);
} catch (ProvisionException e) {
LOG.error("Guice error", e);
annotateProvisionException(e);
System.exit(-1);
return;
} catch (Exception e) {
LOG.error("Unexpected exception", e);
System.exit(-1);
return;
}
Runtime.getRuntime().addShutdownHook(new Thread(injector.getInstance(shutdownHook())));
// propagate default size to input plugins
MessageInput.setDefaultRecvBufferSize(configuration.getUdpRecvBufferSizes());
// Start services.
final ServiceManagerListener serviceManagerListener =
injector.getInstance(ServiceManagerListener.class);
serviceManager.addListener(serviceManagerListener);
try {
serviceManager.startAsync().awaitHealthy();
} catch (Exception e) {
try {
serviceManager
.stopAsync()
.awaitStopped(configuration.getShutdownTimeout(), TimeUnit.MILLISECONDS);
} catch (TimeoutException timeoutException) {
LOG.error("Unable to shutdown properly on time. {}", serviceManager.servicesByState());
}
LOG.error("Graylog startup failed. Exiting. Exception was:", e);
System.exit(-1);
}
LOG.info("Services started, startup times in ms: {}", serviceManager.startupTimes());
activityWriter.write(new Activity("Started up.", Main.class));
LOG.info("Graylog " + commandName + " up and running.");
// Block forever.
try {
Thread.currentThread().join();
} catch (InterruptedException e) {
return;
}
}
