开源网络流量与日志分析-ELK+elasticflow

systemctl stop firewalld
sed -i 's/enforcing/disabled/g' /etc/selinux/config
setenforce 0
yum -y install vim bash-c* net-tools lrzsz wget unzip gcc gcc-c++ epel-release tree 
yum -y install java-openjdk  java-1.8.0-openjdk-devel

wget -c http://itityunwei.cn/tools/elastiflow/elasticsearch-7.8.1-x86_64.rpm
wget -c http://itityunwei.cn/tools/elastiflow/logstash-7.8.1.rpm
wget -c http://itityunwei.cn/tools/elastiflow/kibana-7.8.1-x86_64.rpm
rpm -ivh elasticsearch-7.8.1-x86_64.rpm logstash-7.8.1.rpm kibana-7.8.1-x86_64.rpm
systemctl daemon-reload
systemctl enable elasticsearch.service
systemctl enable kibana.service
systemctl enable logstash.service

vim /etc/elasticsearch/elasticsearch.yml
cluster.name: elk
node.name: node-1
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
--------------------------------------------------------------------------------------
vim /etc/elasticsearch/jvm.options
-Xms4g
-Xmx8g
--------------------------------------------------------------------------------------
vim /etc/logstash/jvm.options
-Xms4g
-Xmx8g
--------------------------------------------------------------------------------------
vim /etc/logstash/startup.options
JAVACMD=/usr/bin/java
--------------------------------------------------------------------------------------
vim /etc/kibana/kibana.yml
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://localhost:9200"]
i18n.locale: "zh-CN"
--------------------------------------------------------------------------------------
systemctl restart elasticsearch.service
systemctl restart kibana.service

/usr/share/logstash/bin/logstash-plugin install logstash-codec-sflow
/usr/share/logstash/bin/logstash-plugin install logstash-codec-netflow
/usr/share/logstash/bin/logstash-plugin install logstash-input-udp
/usr/share/logstash/bin/logstash-plugin install logstash-input-tcp
/usr/share/logstash/bin/logstash-plugin install logstash-filter-dns
/usr/share/logstash/bin/logstash-plugin install logstash-filter-geoip
/usr/share/logstash/bin/logstash-plugin install logstash-filter-translate
/usr/share/logstash/bin/system-install

chown -R kibana:kibana /etc/kibana
chown -R kibana:kibana /usr/share/kibana
chown -R kibana:kibana /etc/default/kibana
chown -R elasticsearch:elasticsearch /etc/elasticsearch
chown -R elasticsearch:elasticsearch /usr/share/elasticsearch
chown -R elasticsearch:elasticsearch /var/lib/elasticsearch
chown -R elasticsearch:elasticsearch /var/log/elasticsearch
chown -R elasticsearch:elasticsearch /etc/sysconfig/elasticsearch
chown -R logstash:logstash /etc/logstash
chown -R logstash:logstash /usr/share/logstash
chown -R logstash:logstash /var/lib/logstash
chown -R logstash:logstash /var/log/logstash
chown -R logstash:logstash /etc/default/logstash

cd /usr/local/src
wget -c http://itityunwei.cn/tools/elastiflow/elastiflow.tar.gz
tar -zxvf elastiflow.tar.gz
cd /usr/local/src/elastiflow
cp -r logstash/elastiflow /etc/logstash/
cp -r logstash.service.d /etc/systemd/system/
chown -R logstash:logstash /etc/logstash/elastiflow
--------------------------------------------------------
vim /etc/logstash/pipelines.yml
path.config: "/etc/logstash/elastiflow/conf.d/*.conf"
--------------------------------------------------------
systemctl daemon-reload
systemctl enable logstash
systemctl start logstash
systemctl restart logstash

cd /etc/logstash/conf.d
vim network.conf
------写入以下内容--------
input {
    udp {
        port => 3500
        type => "network"
    }
}
filter {
 if [type] == "network" {
        grok {
            match => {
                "message" => "<%{BASE10NUM:syslog_pri}>(?<switchtime>.*) %{DATA:hostname} %{DATA:ddModuleName}/%{POSINT:severity}/%{DATA:Brief}:%{GREEDYDATA:message}"
            }
            remove_field => [ "timestamp" ]
            add_field => {
                "severity_code" => "%{severity}"
            }
            overwrite => ["message"]
        }
  }
  mutate {
        gsub => [
            "severity", "0", "Emergency",
            "severity", "1", "Alert",
            "severity", "2", "Critical",
            "severity", "3", "Error",
            "severity", "4", "Warning",
            "severity", "5", "Notice",
            "severity", "6", "Informational",
            "severity", "7", "Debug"
        ]
    }
}
output {
  #stdout { codec => rubydebug }
  if [type] == "network" {
        elasticsearch {
                hosts => ["http://127.0.0.1:9200"]
                index => "filebeat-syslog-%{+YYYY.MM.dd}"
#                ssl => true
#               cacert => "/etc/logstash/certs/ca.crt"
                user => "elastic"
                password => "xxxxx"

        }
  }
}
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/network.conf     ###运行

华为：
sflow agen ip  交换机IP 
sflow collector 1 ip 服务器IP 
sflow collector 1
int g0/0/1
sflow flow-sampling inbound
sflow flow-sampling outbound
sflow flow-sampling collector 1
sflow flow-sampling rate 256 
-----------------------------------------
info-center loghost 交换机IP
info-center loghost 服务器IP port 3500

思科：
flow record flow_record
 match ipv4 source address
 match ipv4 destination address
 match ipv4 protocol
 match transport source-port
 match transport destination-port
 match ipv4 tos
 match interface input
 collect interface output
 collect counter bytes layer2 long
 collect counter packets long

flow exporter flow_export
 destination 服务器IP
 transport udp 2055
 template data timeout 60

flow monitor flow_monitor
 exporter flow_export
 cache timeout active 60
 record flow_record

int g0/0/1
ip flow monitor flow_monitor input

华三：
sflow agent ip 交换机IP
sflow source ip 交换机IP
sflow collector 1 ip 服务器IP 
int g1/0/1
 sflow flow collector 1
 sflow sampling-rate 4000
 sflow counter collector 1
 sflow counter interval 120
-----------------------------------------
info-center loghost 交换机IP
info-center loghost 服务器IP port 3500