Elastic Note

Reading time ~2 minutes

Elasticsearch

$ sudo apt-get install snmp snmpd

Logstash

Input

All the plugins are configured in input{}

input {
    ...
    file {
        ...
    }
    ...
}

General configuration for input file

file {
    # Use "/"" instead of "\" in Windows in order to mach all files
    # The value has to be type of array if it is more than one file
    path => ["C:/ertmslog/rbc/out/*"]
    # Start parsing from the beginning
    start_position => "beginning"
    # sincedb_path is "/dev/null" for for Linux and "Nul" for Windows in order not to remember the status of input files.
    # sincedb_path => "/dev/null"
    # sincedb_path => "Nul"
    sincedb_path => "C:/ertmslog/.sincedb_ertms_log_rbc"
    # Set time longer for testing, the default value is one day in seconds
    ignore_older => 8640000
    type => "rbc"
}

Plain text input for example txt file and also csv file

file {
    ...
    codec => plain {
        charset => "ISO-8859-1"
    }
}

Multiline input for example xml file

file {
    ...
    codec => multiline {
        pattern => "^<packet>"
        negate => "true"
        what => "previous"
        charset => "ISO-8859-1"
    }
}

Filter

All the plugins are configured in filter{}

filter {
    ...
    file {
        ...
    }
    ...
}

Parse csv

csv {
    # Column names can be redefined, i.e can be different from source
    columns => ["col1", "col2", "col3"]
    separator => ";"
}

Parse xml

xml {
    source => "message"
    force_array => "false"
    store_xml => "false"
    xpath => [
        "//proto[@name='subset026']", "proto",
        "//field[@name='timestamp']/@show", "timestamp_tmp",
        "//field[@name='subset026.nid_message']/@show", "nid_message"

    ]
    remove_field => ["message", "@version", "path", "host"]
}

Replace characters in a field

mutate {
    gsub => [
        "message", "\n", "",
        "message", "\r", ""
   ]
}

Replace field by field

mutate {
    replace => {
        "timestamp_tmp" => "%{[timestamp_tmp][0]}"
        "nid_message" => "%{[nid_message][0]}"
     }
}

Parse pattern in a field

grok {
    # Handle more matches
    break_on_match => false
    match => {
        "timestamp_tmp" => "%{GREEDYDATA:timestamp} Vest-Europa (%{GREEDYDATA})"
    }
    match => {
        "timestamp_tmp" => "%{GREEDYDATA:timestamp} W. Europe Daylight Time"
    }
}

http://grokdebug.herokuapp.com/ can be used for debug

Convert data type

mutate {
    convert => {
        "km" => "float"
    }
}

Calculation

ruby {
    code => "event['meter'] = event['km'] * 1000;"
}

Parse date

date {
    locale => "en"
    match => ["timestamp", "MMM d, yyyy HH:mm:ss.SSSSSSSSS", "MMM  d, yyyy HH:mm:ss.SSSSSSSSS"]
    timezone => "UTC"
    target => "timestamp"
    remove_field => ["timestamp_tmp"]
}

Set timezone UTC first for testing if there are some time difference

Drop if you do not want

# For example
if "_grokparsefailure" in [tags] {
    drop { }
}

Output

All the plugins are configured in output{}

General configuration for output

output {
    if [type] == "atb_coordinate" or [type] == "atb_coordinate_2" {
        # Output to Elasticsearch
        elasticsearch {
            hosts => ["localhost:9202"]
            index => "ertms-static-atb"
            document_type => "coordinate"
        }
        # Output to terminal, this can be used for testing
        stdout { }
    }
}

竟然无法拒绝你的打赏

微信支付

专业处理中国护照照片一次通过

专业处理中国护照照片一次通过 Continue reading

Python Notes

Published on April 01, 2021

Scrum Training Notes

Published on December 01, 2020