Commit f8b023f8 authored by mohammad.salama's avatar mohammad.salama

Added Logging (Logstash - ELK - Kibana) and Service Tracing with Visualization (Zipkin)

parent 1e4b3470
...@@ -35,6 +35,12 @@ ...@@ -35,6 +35,12 @@
<groupId>org.springframework.cloud</groupId> <groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-netflix-eureka-client</artifactId> <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.30</version>
<scope>compile</scope>
</dependency>
</dependencies> </dependencies>
<dependencyManagement> <dependencyManagement>
<dependencies> <dependencies>
...@@ -65,6 +71,16 @@ ...@@ -65,6 +71,16 @@
<artifactId>gson</artifactId> <artifactId>gson</artifactId>
<version>2.8.9</version> <version>2.8.9</version>
</dependency> </dependency>
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>7.2</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
......
package org.example; package org.example;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@RestController @RestController
@Slf4j
@RequestMapping("/flights-db") @RequestMapping("/flights-db")
public class FlightDB public class FlightDB
{ {
...@@ -23,6 +26,7 @@ public class FlightDB ...@@ -23,6 +26,7 @@ public class FlightDB
@RequestMapping("/{flightID}") @RequestMapping("/{flightID}")
public static Flight getFlight(@PathVariable("flightID") String flightID) public static Flight getFlight(@PathVariable("flightID") String flightID)
{ {
log.info("Query on Flight with ID : " + flightID);
if (flightsDB.containsKey(flightID)) return flightsDB.get(flightID); if (flightsDB.containsKey(flightID)) return flightsDB.get(flightID);
return new Flight("",new Date(),""); return new Flight("",new Date(),"");
} }
......
package org.example; package org.example;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
...@@ -7,6 +8,7 @@ import org.springframework.web.bind.annotation.RestController; ...@@ -7,6 +8,7 @@ import org.springframework.web.bind.annotation.RestController;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@Slf4j
@RestController @RestController
@RequestMapping("/users-db") @RequestMapping("/users-db")
public class UserDB public class UserDB
...@@ -22,6 +24,7 @@ public class UserDB ...@@ -22,6 +24,7 @@ public class UserDB
@RequestMapping("/{userID}") @RequestMapping("/{userID}")
public static User getUser(@PathVariable("userID")String userID) public static User getUser(@PathVariable("userID")String userID)
{ {
log.info("Query on User with ID : " + userID);
System.out.println(usersDB.get(userID)); System.out.println(usersDB.get(userID));
if (usersDB.containsKey(userID)) return usersDB.get(userID); if (usersDB.containsKey(userID)) return usersDB.get(userID);
return null; return null;
......
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %msg%n</pattern>
<charset>utf8</charset>
</encoder>
</appender>
<appender name="FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>logs/catalog.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>logs/catalog.%d{yyyy-MM-dd}.log</fileNamePattern>
<maxHistory>10</maxHistory>
<totalSizeCap>1GB</totalSizeCap>
</rollingPolicy>
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %msg%n</pattern>
</encoder>
</appender>
<appender name="STASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<destination>localhost:5000</destination>
<encoder
class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<mdc />
<context />
<logLevel />
<loggerName />
<pattern>
<pattern>
{
"appName": "DataBase-service"
}
</pattern>
</pattern>
<threadName />
<message />
<logstashMarkers />
<stackTrace />
</providers>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE" />
<appender-ref ref="FILE" />
<appender-ref ref="STASH" />
</root>
</configuration>
\ No newline at end of file
...@@ -39,6 +39,16 @@ ...@@ -39,6 +39,16 @@
<type>pom</type> <type>pom</type>
<scope>import</scope> <scope>import</scope>
</dependency> </dependency>
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>7.2</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
</dependencies> </dependencies>
<dependencyManagement> <dependencyManagement>
......
...@@ -37,3 +37,17 @@ Configure the Grafana. ...@@ -37,3 +37,17 @@ Configure the Grafana.
- Access "home" - Access "home"
- Import dashboard - Import dashboard
- Upload dashboard.json from /docker - Upload dashboard.json from /docker
## Centralize spring boot log to ELK Elasticsearch, Logstash, Kibana
### Step 1
Use docker-compose to start ELK servers.
- In the root folder
```sh
docker-compose -f docker-compose-elk.yml up
```
### Step 2
Check the kibana server.
- Open http://localhost:5601
...@@ -47,6 +47,18 @@ ...@@ -47,6 +47,18 @@
<artifactId>WeatherService</artifactId> <artifactId>WeatherService</artifactId>
<version>0.0.1-SNAPSHOT</version> <version>0.0.1-SNAPSHOT</version>
<scope>compile</scope> <scope>compile</scope>
</dependency>
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>7.2</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency> </dependency>
</dependencies> </dependencies>
......
package org.example; package org.example;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
...@@ -11,6 +12,7 @@ import org.springframework.web.client.RestTemplate; ...@@ -11,6 +12,7 @@ import org.springframework.web.client.RestTemplate;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@Slf4j
@RestController @RestController
@RequestMapping("/users") @RequestMapping("/users")
public class UserService public class UserService
...@@ -25,11 +27,15 @@ public class UserService ...@@ -25,11 +27,15 @@ public class UserService
@RequestMapping("/{userID}") @RequestMapping("/{userID}")
public List<Flight> getFlights(@PathVariable("userID") String userID) public List<Flight> getFlights(@PathVariable("userID") String userID)
{ {
log.info("Query on Flights for User with ID : " + userID);
User user = restTemplate.getForObject(UserDBURL+userID , User.class); User user = restTemplate.getForObject(UserDBURL+userID , User.class);
if (user == null) return new ArrayList<>(); if (user == null)
{
log.warn("No User With ID : " + userID);
return new ArrayList<>();
}
System.out.println("Welcome : " + user.getName()); System.out.println("Welcome : " + user.getName());
List<Flight> ans = new ArrayList<>(); List<Flight> ans = new ArrayList<>();
/// System.out.println("Ans size is " + user.getFlights().size());
for (String flightID : user.getFlights()) for (String flightID : user.getFlights())
{ {
...@@ -43,8 +49,14 @@ public class UserService ...@@ -43,8 +49,14 @@ public class UserService
@RequestMapping("/weather/{flightID}") @RequestMapping("/weather/{flightID}")
public Weather getWeather(@PathVariable("flightID") String flightID) public Weather getWeather(@PathVariable("flightID") String flightID)
{ {
log.info("Query on Weather and Forecast for Destination of Flight : " + flightID);
Flight flight = restTemplate.getForObject(FlightDBURL+flightID , Flight.class); Flight flight = restTemplate.getForObject(FlightDBURL+flightID , Flight.class);
if (flight.getDestination().isEmpty() ) return new Weather(); if (flight.getDestination().isEmpty() )
{
log.warn("No Flight with ID : " + flightID);
return new Weather();
}
String city = flight.getDestination(); String city = flight.getDestination();
Weather weather = restTemplate.getForObject(WeatherURL+city , Weather.class); Weather weather = restTemplate.getForObject(WeatherURL+city , Weather.class);
......
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %msg%n</pattern>
<charset>utf8</charset>
</encoder>
</appender>
<appender name="FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>logs/catalog.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>logs/catalog.%d{yyyy-MM-dd}.log</fileNamePattern>
<maxHistory>10</maxHistory>
<totalSizeCap>1GB</totalSizeCap>
</rollingPolicy>
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %msg%n</pattern>
</encoder>
</appender>
<appender name="STASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<destination>localhost:5000</destination>
<encoder
class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<mdc />
<context />
<logLevel />
<loggerName />
<pattern>
<pattern>
{
"appName": "User-service"
}
</pattern>
</pattern>
<threadName />
<message />
<logstashMarkers />
<stackTrace />
</providers>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE" />
<appender-ref ref="FILE" />
<appender-ref ref="STASH" />
</root>
</configuration>
\ No newline at end of file
...@@ -49,6 +49,16 @@ ...@@ -49,6 +49,16 @@
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>7.2</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
</dependencies> </dependencies>
<dependencyManagement> <dependencyManagement>
......
package org.example; package org.example;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
...@@ -10,6 +11,7 @@ import org.springframework.web.client.RestTemplate; ...@@ -10,6 +11,7 @@ import org.springframework.web.client.RestTemplate;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@Slf4j
@RestController @RestController
@RequestMapping("/users") @RequestMapping("/users")
public class UserService public class UserService
...@@ -24,11 +26,15 @@ public class UserService ...@@ -24,11 +26,15 @@ public class UserService
@RequestMapping("/{userID}") @RequestMapping("/{userID}")
public List<Flight> getFlights(@PathVariable("userID") String userID) public List<Flight> getFlights(@PathVariable("userID") String userID)
{ {
log.info("Query on Flights for User with ID : " + userID);
User user = restTemplate.getForObject(UserDBURL+userID , User.class); User user = restTemplate.getForObject(UserDBURL+userID , User.class);
if (user == null) return new ArrayList<>(); if (user == null)
{
log.warn("No User With ID : " + userID);
return new ArrayList<>();
}
System.out.println("Welcome : " + user.getName()); System.out.println("Welcome : " + user.getName());
List<Flight> ans = new ArrayList<>(); List<Flight> ans = new ArrayList<>();
/// System.out.println("Ans size is " + user.getFlights().size());
for (String flightID : user.getFlights()) for (String flightID : user.getFlights())
{ {
...@@ -42,8 +48,14 @@ public class UserService ...@@ -42,8 +48,14 @@ public class UserService
@RequestMapping("/weather/{flightID}") @RequestMapping("/weather/{flightID}")
public Weather getWeather(@PathVariable("flightID") String flightID) public Weather getWeather(@PathVariable("flightID") String flightID)
{ {
log.info("Query on Weather and Forecast for Destination of Flight : " + flightID);
Flight flight = restTemplate.getForObject(FlightDBURL+flightID , Flight.class); Flight flight = restTemplate.getForObject(FlightDBURL+flightID , Flight.class);
if (flight.getDestination().isEmpty() ) return new Weather(); if (flight.getDestination().isEmpty() )
{
log.warn("No Flight with ID : " + flightID);
return new Weather();
}
String city = flight.getDestination(); String city = flight.getDestination();
Weather weather = restTemplate.getForObject(WeatherURL+city , Weather.class); Weather weather = restTemplate.getForObject(WeatherURL+city , Weather.class);
......
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %msg%n</pattern>
<charset>utf8</charset>
</encoder>
</appender>
<appender name="FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>logs/catalog.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>logs/catalog.%d{yyyy-MM-dd}.log</fileNamePattern>
<maxHistory>10</maxHistory>
<totalSizeCap>1GB</totalSizeCap>
</rollingPolicy>
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %msg%n</pattern>
</encoder>
</appender>
<appender name="STASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<destination>localhost:5000</destination>
<encoder
class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<mdc />
<context />
<logLevel />
<loggerName />
<pattern>
<pattern>
{
"appName": "User-service-Replica"
}
</pattern>
</pattern>
<threadName />
<message />
<logstashMarkers />
<stackTrace />
</providers>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE" />
<appender-ref ref="FILE" />
<appender-ref ref="STASH" />
</root>
</configuration>
\ No newline at end of file
...@@ -59,6 +59,16 @@ ...@@ -59,6 +59,16 @@
<artifactId>jettison</artifactId> <artifactId>jettison</artifactId>
<version>1.1</version> <version>1.1</version>
</dependency> </dependency>
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>7.2</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
</dependencies> </dependencies>
<dependencyManagement> <dependencyManagement>
<dependencies> <dependencies>
......
package org.example; package org.example;
import lombok.extern.slf4j.Slf4j;
import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONException;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
...@@ -10,7 +11,7 @@ import org.springframework.web.client.RestTemplate; ...@@ -10,7 +11,7 @@ import org.springframework.web.client.RestTemplate;
import java.io.IOException; import java.io.IOException;
@Slf4j
@RestController @RestController
@RequestMapping("/weather") @RequestMapping("/weather")
public class WeatherService public class WeatherService
...@@ -24,10 +25,12 @@ public class WeatherService ...@@ -24,10 +25,12 @@ public class WeatherService
String prefURL = "http://api.weatherapi.com/v1/current.json?key="+ myAPIKey + "&q="; String prefURL = "http://api.weatherapi.com/v1/current.json?key="+ myAPIKey + "&q=";
String sufURL = "&aqi=yes"; String sufURL = "&aqi=yes";
@RequestMapping("/{city}") @RequestMapping("/{city}")
public Weather getWeather(@PathVariable("city") String city) throws IOException { public Weather getWeather(@PathVariable("city") String city) throws IOException
System.out.println("My Api = " + myAPIKey); {
if (city.isEmpty()) if (city.isEmpty())
{ {
log.warn("Request With Empty City");
System.out.println("City is Empty"); System.out.println("City is Empty");
return new Weather(); return new Weather();
} }
...@@ -38,8 +41,7 @@ public class WeatherService ...@@ -38,8 +41,7 @@ public class WeatherService
} catch (JSONException e) } catch (JSONException e)
{ {
System.out.println("EERRRERERER"); log.error("JSON Parsing Error in External API Request for City : " + city);
//log
} }
return weather; return weather;
} }
......
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %msg%n</pattern>
<charset>utf8</charset>
</encoder>
</appender>
<appender name="FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>logs/catalog.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>logs/catalog.%d{yyyy-MM-dd}.log</fileNamePattern>
<maxHistory>10</maxHistory>
<totalSizeCap>1GB</totalSizeCap>
</rollingPolicy>
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %msg%n</pattern>
</encoder>
</appender>
<appender name="STASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<destination>localhost:5000</destination>
<encoder
class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<mdc />
<context />
<logLevel />
<loggerName />
<pattern>
<pattern>
{
"appName": "weather-service"
}
</pattern>
</pattern>
<threadName />
<message />
<logstashMarkers />
<stackTrace />
</providers>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE" />
<appender-ref ref="FILE" />
<appender-ref ref="STASH" />
</root>
</configuration>
\ No newline at end of file
version: "3"
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:8.3.3
container_name: elasticsearch
environment:
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "discovery.type=single-node"
- xpack.security.enabled=false
ports:
- "9200:9200"
volumes:
- elasticsearch_data:/usr/share/elasticsearch/data
networks:
- elastic
kibana:
image: docker.elastic.co/kibana/kibana:8.3.3
container_name: kibana
ports:
- "5601:5601"
environment:
ELASTICSEARCH_URL: http://elasticsearch:9200
ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
depends_on:
- elasticsearch
networks:
- elastic
logstash:
image: docker.elastic.co/logstash/logstash:8.3.3
container_name: logstash
volumes:
- ./docker/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
- ./docker/logstash/pipeline:/usr/share/logstash/pipeline:ro
ports:
- "5044:5044"
- "5000:5000/tcp"
- "5000:5000/udp"
- "9600:9600"
environment:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
networks:
- elastic
depends_on:
- elasticsearch
zipkin:
image: openzipkin/zipkin
container_name: zipkin
ports:
- "9411:9411"
networks:
- elastic
networks:
elastic:
driver: bridge
volumes:
elasticsearch_data:
driver: local
\ No newline at end of file
http.host: "0.0.0.0"
path.config: /usr/share/logstash/pipeline
xpack.monitoring.elasticsearch.hosts: [ "elasticsearch:9200" ]
\ No newline at end of file
input {
tcp{
port => 5000
codec => json
}
}
output {
elasticsearch {
hosts => "elasticsearch:9200"
index => "springboot-%{app}"
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment