Forward Logs to logstash over a socket

This is a sample configuration for forwarding logs to logstash over a socket with Springboot application




//Add the properties file in classpath
logstash.properties
logstash.host=logstash-server logstash.port=5000 logstash.queue-size=512;

 
 

//Create Bean populated from logstash.properties file
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.ConfigurationProperties;
import org.springframework.context.annotation.PropertySource;

@Configuration;
@PropertySource("classpath:logstash.properties");
@ConfigurationProperties(prefix="logstash"); 

public class LogstashProperties{
  private String host;   private Integer port;    private  Integer queueSize;   //  GETTER SETTER follow }




// Configure logging to append logs to logstash

import java.net.InetSocketAddress; import java.util.Iterator; import ch.qos.logback.classic.AsyncAppender; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.boolex.OnMarkerEvaluator; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.LoggerContextListener; import ch.qos.logback.core.Appender; import ch.qos.logback.core.filter.EvaluatorFilter; import ch.qos.logback.core.spi.ContextAwareBase; import ch.qos.logback.core.spi.FilterReply; import net.logstash.logback.appender.LogstashTcpSocketAppender; import net.logstash.logback.encoder.LogstashEncoder; import net.logstash.logback.stacktrace.ShortenedThrowableConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; @Configuration public class MyLoggingConfiguration{ private static final String LOGSTASH_APPENDER_NAME = "LOGSTASH"; private static final String ASYNC_LOGSTASH_APPENDER_NAME = "ASYNC_LOGSTASH";
private final Logger log = LoggerFactory.getLogger(MyLoggingConfiguration.class);
private LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
private final String appName;
private final String serverPort;
private final LogstashProperties logstashProperties;
public MyLoggingConfiguration(@Value("${spring.application.name}") String appName,
@Value("${server.port}") String serverPort, LogstashProperties logstashProperties) {
this.appName = appName;
this.serverPort = serverPort; this.logstashProperties=logstashProperties; } private void addContextListener(LoggerContext context) { LogbackLoggerContextListener loggerContextListener = new LogbackLoggerContextListener(); loggerContextListener.setContext(context); context.addListener(loggerContextListener); } private void addLogstashAppender(LoggerContext context) { log.info("Initializing Logstash logging"); LogstashTcpSocketAppender logstashAppender = new LogstashTcpSocketAppender(); logstashAppender.setName(LOGSTASH_APPENDER_NAME); logstashAppender.setContext(context); String customFields = "{\"app_name\":\"" + appName + "\",\"app_port\":\"" + serverPort + "\"}"; // More documentation is available at: https://github.com/logstash/logstash-logback-encoder LogstashEncoder logstashEncoder=new LogstashEncoder(); // Set the Logstash appender config from logstashProperties properties logstashEncoder.setCustomFields(customFields); // Set the Logstash appender config from logstashProperties properties logstashAppender.addDestinations(
new InetSocketAddress(logstashProperties.getLogging().getLogstash().getHost(),
logstashProperties.getLogging().getLogstash().getPort()));
ShortenedThrowableConverter throwableConverter = new ShortenedThrowableConverter(); throwableConverter.setRootCauseFirst(true); logstashEncoder.setThrowableConverter(throwableConverter); logstashEncoder.setCustomFields(customFields); logstashAppender.setEncoder(logstashEncoder); logstashAppender.start(); // Wrap the appender in an Async appender for performance AsyncAppender asyncLogstashAppender = new AsyncAppender(); asyncLogstashAppender.setContext(context); asyncLogstashAppender.setName(ASYNC_LOGSTASH_APPENDER_NAME); asyncLogstashAppender.setQueueSize(
logstashProperties.getLogging().getLogstash().getQueueSize()); asyncLogstashAppender.addAppender(logstashAppender); asyncLogstashAppender.start(); context.getLogger("ROOT").addAppender(asyncLogstashAppender);
} // Configure a log filter to remove "metrics" logs from all appenders except the
// "LOGSTASH" appender private void setMetricsMarkerLogbackFilter(LoggerContext context) {
log.info("Filtering metrics logs from all appenders except the {} appender",
LOGSTASH_APPENDER_NAME);
OnMarkerEvaluator onMarkerMetricsEvaluator = new OnMarkerEvaluator(); onMarkerMetricsEvaluator.setContext(context); onMarkerMetricsEvaluator.addMarker("metrics"); onMarkerMetricsEvaluator.start(); EvaluatorFilter<ILoggingEvent> metricsFilter = new EvaluatorFilter<>(); metricsFilter.setContext(context); metricsFilter.setEvaluator(onMarkerMetricsEvaluator); metricsFilter.setOnMatch(FilterReply.DENY); metricsFilter.start(); for (ch.qos.logback.classic.Logger logger : context.getLoggerList()) { for (Iterator<Appender<ILoggingEvent>> it = logger.iteratorForAppenders(); it.hasNext();) { Appender<ILoggingEvent> appender = it.next(); if (!appender.getName().equals(ASYNC_LOGSTASH_APPENDER_NAME)) { log.debug("Filter metrics logs from the {} appender", appender.getName()); appender.setContext(context); appender.addFilter(metricsFilter); appender.start(); } } }
} /** * Logback configuration is achieved by configuration file and API. * When configuration file change is detected, the configuration is reset. * This listener ensures that the programmatic configuration is also re-applied after reset. */ class LogbackLoggerContextListener extends ContextAwareBase
implements LoggerContextListener { @Override public boolean isResetResistant() { return true; } @Override public void onStart(LoggerContext context) { addLogstashAppender(context); } @Override public void onReset(LoggerContext context) { addLogstashAppender(context); } @Override public void onStop(LoggerContext context) { // Nothing to do. } @Override public void onLevelChange(ch.qos.logback.classic.Logger logger, Level level) { // Nothing to do. } } }

Read More