Elasticsearch
$ sudo apt-get install snmp snmpd
Logstash
Input
All the plugins are configured in input{}
input {
...
file {
...
}
...
}
General configuration for input file
file {
# Use "/"" instead of "\" in Windows in order to mach all files
# The value has to be type of array if it is more than one file
path => ["C:/ertmslog/rbc/out/*"]
# Start parsing from the beginning
start_position => "beginning"
# sincedb_path is "/dev/null" for for Linux and "Nul" for Windows in order not to remember the status of input files.
# sincedb_path => "/dev/null"
# sincedb_path => "Nul"
sincedb_path => "C:/ertmslog/.sincedb_ertms_log_rbc"
# Set time longer for testing, the default value is one day in seconds
ignore_older => 8640000
type => "rbc"
}
Plain text input for example txt file and also csv file
file {
...
codec => plain {
charset => "ISO-8859-1"
}
}
Multiline input for example xml file
file {
...
codec => multiline {
pattern => "^<packet>"
negate => "true"
what => "previous"
charset => "ISO-8859-1"
}
}
Filter
All the plugins are configured in filter{}
filter {
...
file {
...
}
...
}
Parse csv
csv {
# Column names can be redefined, i.e can be different from source
columns => ["col1", "col2", "col3"]
separator => ";"
}
Parse xml
xml {
source => "message"
force_array => "false"
store_xml => "false"
xpath => [
"//proto[@name='subset026']", "proto",
"//field[@name='timestamp']/@show", "timestamp_tmp",
"//field[@name='subset026.nid_message']/@show", "nid_message"
]
remove_field => ["message", "@version", "path", "host"]
}
Replace characters in a field
mutate {
gsub => [
"message", "\n", "",
"message", "\r", ""
]
}
Replace field by field
mutate {
replace => {
"timestamp_tmp" => "%{[timestamp_tmp][0]}"
"nid_message" => "%{[nid_message][0]}"
}
}
Parse pattern in a field
grok {
# Handle more matches
break_on_match => false
match => {
"timestamp_tmp" => "%{GREEDYDATA:timestamp} Vest-Europa (%{GREEDYDATA})"
}
match => {
"timestamp_tmp" => "%{GREEDYDATA:timestamp} W. Europe Daylight Time"
}
}
http://grokdebug.herokuapp.com/ can be used for debug
Convert data type
mutate {
convert => {
"km" => "float"
}
}
Calculation
ruby {
code => "event['meter'] = event['km'] * 1000;"
}
Parse date
date {
locale => "en"
match => ["timestamp", "MMM d, yyyy HH:mm:ss.SSSSSSSSS", "MMM d, yyyy HH:mm:ss.SSSSSSSSS"]
timezone => "UTC"
target => "timestamp"
remove_field => ["timestamp_tmp"]
}
Set timezone UTC first for testing if there are some time difference
Drop if you do not want
# For example
if "_grokparsefailure" in [tags] {
drop { }
}
Output
All the plugins are configured in output{}
General configuration for output
output {
if [type] == "atb_coordinate" or [type] == "atb_coordinate_2" {
# Output to Elasticsearch
elasticsearch {
hosts => ["localhost:9202"]
index => "ertms-static-atb"
document_type => "coordinate"
}
# Output to terminal, this can be used for testing
stdout { }
}
}
竟然无法拒绝你的打赏
