Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Sign in
Toggle navigation
F
FMS_Project
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
bashar.hussein
FMS_Project
Commits
cef61db6
Commit
cef61db6
authored
Jul 07, 2023
by
Bashar
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Configured ElasticSearch
parent
03ea8ffa
Changes
14
Show whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
122 additions
and
84 deletions
+122
-84
elasticConfig.java
...m/example/elasticJavaApi/configuration/elasticConfig.java
+6
-1
elasticConfig.class
.../example/elasticJavaApi/configuration/elasticConfig.class
+0
-0
ProcessingApplication.java
...in/java/com/example/Processing/ProcessingApplication.java
+28
-2
ProcessedTrap.java
...n/java/com/example/Processing/entities/ProcessedTrap.java
+9
-8
ElasticService.java
.../java/com/example/Processing/services/ElasticService.java
+73
-0
application.properties
...line/Processing/src/main/resources/application.properties
+2
-0
index_template.json
...e/Processing/src/main/resources/utils/index_template.json
+2
-2
settings.json
...ipeline/Processing/src/main/resources/utils/settings.json
+1
-1
TrapData.java
...main/java/com/example/SnmpReciever/enitites/TrapData.java
+1
-1
TrapData.class
.../classes/com/example/SnmpReciever/enitites/TrapData.class
+0
-0
index_template.json
...afkaConsumerSNMP/target/classes/utils/index_template.json
+0
-10
mappings.json
...line/kafkaConsumerSNMP/target/classes/utils/mappings.json
+0
-13
policy.json
...peline/kafkaConsumerSNMP/target/classes/utils/policy.json
+0
-39
settings.json
...line/kafkaConsumerSNMP/target/classes/utils/settings.json
+0
-7
No files found.
Elastic_Configs/elasticJavaApi/src/main/java/com/example/elasticJavaApi/configuration/elasticConfig.java
View file @
cef61db6
...
@@ -27,6 +27,7 @@ import co.elastic.clients.transport.ElasticsearchTransport;
...
@@ -27,6 +27,7 @@ import co.elastic.clients.transport.ElasticsearchTransport;
import
co.elastic.clients.transport.rest_client.RestClientTransport
;
import
co.elastic.clients.transport.rest_client.RestClientTransport
;
import
com.example.elasticJavaApi.entities.ProcessedTrap
;
import
com.example.elasticJavaApi.entities.ProcessedTrap
;
import
com.example.elasticJavaApi.entities.SeverityLevel
;
import
com.example.elasticJavaApi.entities.SeverityLevel
;
import
com.example.elasticJavaApi.entities.VarBind
;
import
com.fasterxml.jackson.core.JsonProcessingException
;
import
com.fasterxml.jackson.core.JsonProcessingException
;
import
com.fasterxml.jackson.databind.JsonNode
;
import
com.fasterxml.jackson.databind.JsonNode
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
...
@@ -132,7 +133,11 @@ public class elasticConfig {
...
@@ -132,7 +133,11 @@ public class elasticConfig {
addComponent
(
workingPath
+
"/settings.json"
,
"bashar_setting"
);
addComponent
(
workingPath
+
"/settings.json"
,
"bashar_setting"
);
addComponent
(
workingPath
+
"/mappings.json"
,
"bashar_mapping"
);
addComponent
(
workingPath
+
"/mappings.json"
,
"bashar_mapping"
);
addIndexTemplate
(
workingPath
+
"/index_template.json"
,
"bashar_template"
);
addIndexTemplate
(
workingPath
+
"/index_template.json"
,
"bashar_template"
);
ProcessedTrap
p
=
new
ProcessedTrap
(
"1.3.2.5.4.6.7"
,
"192.168.25.254"
,
6
,
201
,
1569845288
,
SeverityLevel
.
ERROR
,
new
ArrayList
<>(),
new
GeoPoint
(
Math
.
random
(),
Math
.
random
()));
ArrayList
arrayList
=
new
ArrayList
<
VarBind
>();
VarBind
v
=
new
VarBind
(
"1.1.1.1"
,
"3232"
);
arrayList
.
add
(
v
);
arrayList
.
add
(
v
);
ProcessedTrap
p
=
new
ProcessedTrap
(
"1.3.2.5.4.6.7"
,
"192.168.25.254"
,
6
,
201
,
1569845288
,
SeverityLevel
.
ERROR
,
arrayList
,
new
GeoPoint
(
Math
.
random
(),
Math
.
random
()));
ObjectMapper
ob
=
new
ObjectMapper
();
ObjectMapper
ob
=
new
ObjectMapper
();
//addDataStream("bashar-data-stream-3");
//addDataStream("bashar-data-stream-3");
try
{
try
{
...
...
Elastic_Configs/elasticJavaApi/target/classes/com/example/elasticJavaApi/configuration/elasticConfig.class
View file @
cef61db6
No preview for this file type
SNMP_Pipeline/Processing/src/main/java/com/example/Processing/ProcessingApplication.java
View file @
cef61db6
...
@@ -3,19 +3,28 @@ package com.example.Processing;
...
@@ -3,19 +3,28 @@ package com.example.Processing;
import
com.example.Processing.entities.EnrichedTrap
;
import
com.example.Processing.entities.EnrichedTrap
;
import
com.example.Processing.entities.ProcessedTrap
;
import
com.example.Processing.entities.ProcessedTrap
;
import
com.example.Processing.entities.SeverityLevel
;
import
com.example.Processing.entities.SeverityLevel
;
import
com.example.Processing.services.ElasticService
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
jakarta.annotation.PostConstruct
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.boot.SpringApplication
;
import
org.springframework.boot.SpringApplication
;
import
org.springframework.boot.autoconfigure.SpringBootApplication
;
import
org.springframework.boot.autoconfigure.SpringBootApplication
;
import
org.springframework.kafka.annotation.KafkaListener
;
import
org.springframework.kafka.annotation.KafkaListener
;
import
java.util.List
;
import
java.util.Timer
;
import
java.util.TimerTask
;
@SpringBootApplication
@SpringBootApplication
public
class
ProcessingApplication
{
public
class
ProcessingApplication
{
public
static
void
main
(
String
[]
args
)
{
public
static
void
main
(
String
[]
args
)
{
SpringApplication
.
run
(
ProcessingApplication
.
class
,
args
);
SpringApplication
.
run
(
ProcessingApplication
.
class
,
args
);
}
}
public
static
int
num
=
0
;
public
static
int
num
=
0
;
@Autowired
private
ElasticService
elasticService
;
@KafkaListener
(
topics
=
"EnrichedTrap"
)
@KafkaListener
(
topics
=
"EnrichedTrap"
)
public
void
handleKafkaMessage
(
String
pduJson
)
{
public
void
handleKafkaMessage
(
String
pduJson
)
{
String
json
=
pduJson
;
// Replace with your JSON string
String
json
=
pduJson
;
// Replace with your JSON string
...
@@ -25,11 +34,28 @@ public class ProcessingApplication {
...
@@ -25,11 +34,28 @@ public class ProcessingApplication {
EnrichedTrap
t
=
objectMapper
.
readValue
(
json
,
EnrichedTrap
.
class
);
EnrichedTrap
t
=
objectMapper
.
readValue
(
json
,
EnrichedTrap
.
class
);
//Some Processing: Add this later (Filtering, Prioritizing, Correlation)
//Some Processing: Add this later (Filtering, Prioritizing, Correlation)
ProcessedTrap
t2
=
new
ProcessedTrap
(
t
);
ProcessedTrap
t2
=
new
ProcessedTrap
(
t
);
elasticService
.
addToBulk
(
t2
);
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
System
.
out
.
println
(
e
);
System
.
out
.
println
(
e
);
e
.
printStackTrace
();
e
.
printStackTrace
();
}
}
//System.out.println(num);
//System.out.println(num);
}
}
@PostConstruct
public
void
magic
(){
Timer
timer
=
new
Timer
();
TimerTask
task
=
new
TimerTask
()
{
public
void
run
()
{
// Call your function here
myFunction
();
}
};
// Schedule the task to run every 10 seconds
timer
.
schedule
(
task
,
0
,
3000
);
}
public
void
myFunction
(){
System
.
out
.
println
(
"Hey"
);
elasticService
.
sendBulk
();
}
}
}
SNMP_Pipeline/Processing/src/main/java/com/example/Processing/entities/ProcessedTrap.java
View file @
cef61db6
package
com
.
example
.
Processing
.
entities
;
package
com
.
example
.
Processing
.
entities
;
import
com.fasterxml.jackson.annotation.JsonProperty
;
import
com.fasterxml.jackson.annotation.JsonProperty
;
import
lombok.AllArgsConstructor
;
import
lombok.NoArgsConstructor
;
import
org.springframework.data.elasticsearch.core.geo.GeoPoint
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.List
;
@AllArgsConstructor
@NoArgsConstructor
public
class
ProcessedTrap
{
public
class
ProcessedTrap
{
@JsonProperty
(
"enterprise"
)
@JsonProperty
(
"enterprise"
)
public
String
enterprise
;
public
String
enterprise
;
...
@@ -14,7 +19,7 @@ public class ProcessedTrap {
...
@@ -14,7 +19,7 @@ public class ProcessedTrap {
public
int
genericTrap
;
public
int
genericTrap
;
@JsonProperty
(
"specificTrap"
)
@JsonProperty
(
"specificTrap"
)
public
int
specificTrap
;
public
int
specificTrap
;
@JsonProperty
(
"timestamp"
)
@JsonProperty
(
"
@
timestamp"
)
public
long
timestamp
;
public
long
timestamp
;
@JsonProperty
(
"severity"
)
@JsonProperty
(
"severity"
)
...
@@ -23,11 +28,8 @@ public class ProcessedTrap {
...
@@ -23,11 +28,8 @@ public class ProcessedTrap {
@JsonProperty
(
"variableBindings"
)
@JsonProperty
(
"variableBindings"
)
public
List
<
VarBind
>
variableBindings
=
new
ArrayList
<
VarBind
>();
public
List
<
VarBind
>
variableBindings
=
new
ArrayList
<
VarBind
>();
@JsonProperty
(
"location_X"
)
@JsonProperty
(
"location"
)
public
double
xPoint
;
public
GeoPoint
Point
;
@JsonProperty
(
"location_Y"
)
public
double
yPoint
;
public
ProcessedTrap
(
EnrichedTrap
pdu
)
{
public
ProcessedTrap
(
EnrichedTrap
pdu
)
{
...
@@ -38,7 +40,6 @@ public class ProcessedTrap {
...
@@ -38,7 +40,6 @@ public class ProcessedTrap {
this
.
agentAddress
=
pdu
.
getAgentAddress
().
toString
();
this
.
agentAddress
=
pdu
.
getAgentAddress
().
toString
();
this
.
variableBindings
=
pdu
.
getVariableBindings
();
this
.
variableBindings
=
pdu
.
getVariableBindings
();
this
.
severity
=
pdu
.
getSeverity
();
this
.
severity
=
pdu
.
getSeverity
();
this
.
xPoint
=
Math
.
random
();
this
.
Point
=
new
GeoPoint
(
Math
.
random
(),
Math
.
random
());
this
.
yPoint
=
Math
.
random
();
}
}
}
}
SNMP_Pipeline/Processing/src/main/java/com/example/Processing/services/ElasticService.java
0 → 100644
View file @
cef61db6
package
com
.
example
.
Processing
.
services
;
import
co.elastic.clients.elasticsearch.core.BulkRequest
;
import
co.elastic.clients.elasticsearch.core.BulkResponse
;
import
co.elastic.clients.elasticsearch.core.IndexRequest
;
import
co.elastic.clients.elasticsearch.core.IndexResponse
;
import
com.example.Processing.configuration.elasticConfig
;
import
com.example.Processing.entities.ProcessedTrap
;
import
com.fasterxml.jackson.core.JsonProcessingException
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.stereotype.Service
;
import
java.io.IOException
;
import
java.io.Reader
;
import
java.io.StringReader
;
import
java.util.ArrayList
;
import
java.util.List
;
@Service
public
class
ElasticService
{
@Autowired
private
elasticConfig
elConf
;
private
List
<
ProcessedTrap
>
bulk
=
new
ArrayList
<>();
public
void
addDoc
(
ProcessedTrap
processedTrap
){
ObjectMapper
objectMapper
=
new
ObjectMapper
();
try
{
String
json
=
objectMapper
.
writeValueAsString
(
processedTrap
);
Reader
input
=
new
StringReader
(
json
);
System
.
out
.
println
(
json
);
IndexRequest
request
=
IndexRequest
.
of
(
i
->
i
.
index
(
"traps-data-stream"
)
.
withJson
(
input
)
);
IndexResponse
response
=
null
;
response
=
elConf
.
getElasticClient
().
index
(
request
);
System
.
out
.
println
(
"Indexed with version "
+
response
.
version
());
}
catch
(
Exception
e
)
{
throw
new
RuntimeException
(
e
);
}
}
public
void
addToBulk
(
ProcessedTrap
processedTrap
){
bulk
.
add
(
processedTrap
);
}
public
void
sendBulk
(){
if
(
bulk
.
size
()
==
0
)
{
System
.
out
.
println
(
"Nothing in the bulk"
);
return
;
}
BulkRequest
.
Builder
br
=
new
BulkRequest
.
Builder
();
for
(
ProcessedTrap
s:
bulk
){
br
.
operations
(
op
->
op
.
create
(
cr
->
cr
.
index
(
"traps-data-stream"
).
document
(
s
))
);
}
System
.
out
.
println
(
"Sent "
+
bulk
.
size
());
bulk
.
clear
();
BulkResponse
result
=
null
;
System
.
out
.
println
(
"Hey I am here"
);
try
{
System
.
out
.
println
(
"Hey I am here"
);
result
=
elConf
.
getElasticClient
().
bulk
(
br
.
build
());
System
.
out
.
println
(
result
);
}
catch
(
IOException
e
)
{
System
.
out
.
println
(
e
);
throw
new
RuntimeException
(
e
);
}
}
}
SNMP_Pipeline/Processing/src/main/resources/application.properties
View file @
cef61db6
...
@@ -3,3 +3,5 @@ spring.kafka.template.default-topic=EnrichedTrap
...
@@ -3,3 +3,5 @@ spring.kafka.template.default-topic=EnrichedTrap
spring.kafka.consumer.group-id
=
EnrichedTrapId
spring.kafka.consumer.group-id
=
EnrichedTrapId
spring.kafka.consumer.key-deserializer
=
org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.key-deserializer
=
org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer
=
org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer
=
org.apache.kafka.common.serialization.StringDeserializer
server.port
=
0
workingPath
=
./Processing/src/main/resources/utils
SNMP_Pipeline/Processing/src/main/resources/utils/index_template.json
View file @
cef61db6
{
{
"index_patterns"
:
[
"
bashar
-data-stream*"
],
"index_patterns"
:
[
"
traps
-data-stream*"
],
"data_stream"
:
{
},
"data_stream"
:
{
},
"composed_of"
:
[
"
bashar_setting"
,
"bashar_mapping"
],
"composed_of"
:
[
"
traps_setting"
,
"traps_mapping"
],
"priority"
:
670
,
"priority"
:
670
,
"_meta"
:
{
"_meta"
:
{
"description"
:
"Template for my time series data"
,
"description"
:
"Template for my time series data"
,
...
...
SNMP_Pipeline/Processing/src/main/resources/utils/settings.json
View file @
cef61db6
{
{
"settings"
:
{
"settings"
:
{
"index.lifecycle.name"
:
"
Bashar
"
,
"index.lifecycle.name"
:
"
Traps_Policy
"
,
"index.number_of_shards"
:
1
,
"index.number_of_shards"
:
1
,
"index.number_of_replicas"
:
0
"index.number_of_replicas"
:
0
}
}
...
...
SNMP_Pipeline/SnmpReciever/src/main/java/com/example/SnmpReciever/enitites/TrapData.java
View file @
cef61db6
...
@@ -49,7 +49,7 @@ public class TrapData {
...
@@ -49,7 +49,7 @@ public class TrapData {
this
.
enterprise
=
pdu
.
getEnterprise
().
toString
();
this
.
enterprise
=
pdu
.
getEnterprise
().
toString
();
this
.
genericTrap
=
pdu
.
getGenericTrap
();
this
.
genericTrap
=
pdu
.
getGenericTrap
();
this
.
specificTrap
=
pdu
.
getSpecificTrap
();
this
.
specificTrap
=
pdu
.
getSpecificTrap
();
this
.
timestamp
=
pdu
.
getTimestamp
();
this
.
timestamp
=
pdu
.
getTimestamp
()
*
1000
;
this
.
agentAddress
=
pdu
.
getAgentAddress
().
toString
();
this
.
agentAddress
=
pdu
.
getAgentAddress
().
toString
();
List
<
VariableBinding
>
trapVariableBindings
=
pdu
.
getAll
();
List
<
VariableBinding
>
trapVariableBindings
=
pdu
.
getAll
();
...
...
SNMP_Pipeline/SnmpReciever/target/classes/com/example/SnmpReciever/enitites/TrapData.class
View file @
cef61db6
No preview for this file type
SNMP_Pipeline/kafkaConsumerSNMP/target/classes/utils/index_template.json
deleted
100644 → 0
View file @
03ea8ffa
{
"index_patterns"
:
[
"bashar-data-stream*"
],
"data_stream"
:
{
},
"composed_of"
:
[
"bashar_setting"
,
"bashar_mapping"
],
"priority"
:
670
,
"_meta"
:
{
"description"
:
"Template for my time series data"
,
"my-custom-meta-field"
:
"More arbitrary metadata"
}
}
\ No newline at end of file
SNMP_Pipeline/kafkaConsumerSNMP/target/classes/utils/mappings.json
deleted
100644 → 0
View file @
03ea8ffa
{
"mappings"
:
{
"properties"
:
{
"@timestamp"
:
{
"type"
:
"date"
,
"format"
:
"date_optional_time||epoch_millis"
},
"message"
:
{
"type"
:
"wildcard"
}
}
}
}
\ No newline at end of file
SNMP_Pipeline/kafkaConsumerSNMP/target/classes/utils/policy.json
deleted
100644 → 0
View file @
03ea8ffa
{
"phases"
:
{
"hot"
:
{
"min_age"
:
"0ms"
,
"actions"
:
{
"set_priority"
:
{
"priority"
:
100
},
"rollover"
:
{
"max_primary_shard_size"
:
"50gb"
,
"max_age"
:
"2m"
}
}
},
"warm"
:
{
"min_age"
:
"2m"
,
"actions"
:
{
"set_priority"
:
{
"priority"
:
50
}
}
},
"cold"
:
{
"min_age"
:
"4m"
,
"actions"
:
{
"set_priority"
:
{
"priority"
:
0
}
}
},
"delete"
:
{
"min_age"
:
"6m"
,
"actions"
:
{
"delete"
:
{
}
}
}
}
}
\ No newline at end of file
SNMP_Pipeline/kafkaConsumerSNMP/target/classes/utils/settings.json
deleted
100644 → 0
View file @
03ea8ffa
{
"settings"
:
{
"index.lifecycle.name"
:
"Bashar"
,
"index.number_of_shards"
:
1
,
"index.number_of_replicas"
:
0
}
}
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment