ELK实战之通过TCP收集日志

fjqzyc

一、TCP收集日志使用场景
　　tcp模块的使用场景如下： 有一台服务器A只需要收集一个日志，那么我们就可以不需要在这服务器上安装logstash，我们通过在其他logstash上启用tcp模块，监听某个端口，然后我们在这个服务器A把日志通过nc发送到logstash上即可。

二、标准输出测试TCP模块

[root@linux-node2 ~]# cat /etc/logstash/conf.d/tcp.conf
input {
tcp{
port => "5600"    #监听5600端口
mode => "server"   #模式为server
type => "tcplog"     #类型为tcplog
}
}
output {
stdout {
codec => rubydebug
}
}
#检测配置文件语法：
[root@linux-node2 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tcp.conf -t
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
Configuration OK
#node1节点上安装nc命令，并发送日志到node2。Netcat简称nc，在网络工具中具有“瑞士×××”美誉，其功能实用，是一个简单，可靠的网络工具，可通过TCP或UDP协议传输读写数据，另外还具有很多其他功能。
[root@linux-node1 ~]# yum install -y nc
#通过nc来发送日志
[root@linux-node1 ~]# echo "hello world" | nc 192.168.56.12 5600
#linux-node2终端上查看日志输出信息：
[root@linux-node2 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tcp.conf
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
{
"@timestamp" => 2018-01-02T00:59:49.356Z,
"port" => 57902,
"@version" => "1",
"host" => "linux-node1",
"@metdata" => {
"ip_address" => "192.168.56.11"
},
"message" => "hello world",
"type" => "tcplog"
}
#可以看到linux-node2上有监听5600端口
[root@linux-node2 ~]# netstat -tunlp |grep 5600
tcp6       0      0 :::5600                 :::*                    LISTEN      2301/java           
#还可以将某个文件发送到nc
[root@linux-node1 ~]# nc 192.168.56.12 5600 < /etc/passwd
[root@linux-node2 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tcp.conf
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
"@timestamp" => 2018-01-02T01:00:54.530Z,
"port" => 58134,
"@version" => "1",
"host" => "linux-node1",
"@metdata" => {
"ip_address" => "192.168.56.11"
},
"message" => "root:x:0:0:root:/root:/bin/bash",
"type" => "tcplog"
}
{
"@timestamp" => 2018-01-02T01:00:54.531Z,
"port" => 58134,
"@version" => "1",
"host" => "linux-node1",
"@metdata" => {
"ip_address" => "192.168.56.11"
},
"message" => "bin:x:1:1:bin:/bin:/sbin/nologin",
"type" => "tcplog"
}
......
#也可以通过这种方式伪设备的方式发送日志：（在类unix操作系统中，设备节点并一定要对应物理设备。没有这种对应关系的设备是伪设备。操作系统运用了它们提供的多种功能，tcp只是dev下面众多伪设备当中的一种设备。）
[root@linux-node1 ~]# echo "222" > /dev/tcp/192.168.56.12/5600
[root@linux-node2 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tcp.conf
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
{
"@timestamp" => 2018-01-02T01:26:55.922Z,
"port" => 35576,
"@version" => "1",
"host" => "linux-node1",
"@metdata" => {
"ip_address" => "192.168.56.11"
},
"message" => "222",
"type" => "tcplog"
}
三、配置logstash通过TCP收集输出到elasticsearch

[root@linux-node2 conf.d]# vim tcp.conf
input {
tcp{
port => "5600"
mode => "server"
type => "tcplog"
}
}
output {
elasticsearch {
hosts => ["192.168.56.11:9200"]
index => "tcp-test5612-%{+YYYY.MM.dd}"
}
file {
path => "/tmp/tcp-test5612-%{+YYYY.MM.dd}"
}
}
[root@linux-node2 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tcp.conf -t
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
Configuration OK
[root@linux-node2 conf.d]# systemctl restart logstash
[root@linux-node1 elasticsearch-head]# echo "hello worl" |nc 192.168.56.12 5600
[root@linux-node1 elasticsearch-head]# nc 192.168.56.12 5600 < /etc/passwd

　　HEAD插件查看：


　　Kibana添加索引查看：