0

I have fluentd setup for Kubernetes which is working just fine but when add more sources it fails. I'm trying to get auth.log like this:

 <source>
  @id authlog
  @type tail
  format none
  path /var/log/auth.log
  pos_file /var/log/auth.log.pos
  tag authlog
</source>

It fails with

2019-02-01 12:03:38 +0000 [warn]: dump an error event: error_class=Fluent::Plugin::ConcatFilter::TimeoutError error="Timeout flush: authlog:default" location=nil tag="authlog" time=2019-02-01 12:03:38.825995892 +0000 record={"message"=>"Feb  1 05:01:54 g-k8s-master-0 sshd[32024]: Received disconnect from 10.10.40.2 port 47912:11: disconnected by userFeb  1 05:01:54 g-k8s-master-0 sshd[32024]: Disconnected from 10.10.40.2 port 47912Feb  1 05:01:54 g-k8s-master-0 sshd[31984]: pam_unix(sshd:session): session closed for user paleksicFeb  1 05:01:54 g-k8s-master-0 systemd-logind[1025]: Removed session 1154.Feb  1 05:01:57 g-k8s-master-0 sshd[523]: Accepted publickey for paleksic from 10.10.40.2 port 47942 ssh2: RSA SHA256:rHk7JkW44vqcomA66hsgmX14CAtCE1KytT3/lc4CoDoFeb  1 05:01:57 g-k8s-master-0 sshd[523]: pam_unix(sshd:session): session opened for user paleksic by (uid=0)Feb  1 05:01:57 g-k8s-master-0 systemd: pam_unix(systemd-user:session): session opened for user paleksic by (uid=0)Feb  1 05:01:57 g-k8s-master-0 systemd-logind[1025]: New session 1155 of user paleksic.Feb  1 05:02:32 g-k8s-master-0 sshd[566]: Received disconnect from 10.10.40.2 port 47942:11: disconnected by userFeb  1 05:02:32 g-k8s-master-0 sshd[566]: Disconnected from 10.10.40.2 port 47942Feb  1 05:02:32 g-k8s-master-0 sshd[523]: pam_unix(sshd:session): session closed for user paleksicFeb  1 05:02:32 g-k8s-master-0 systemd-logind[1025]: Removed session 1155.Feb  1 05:02:32 g-k8s-master-0 systemd: pam_unix(systemd-user:session): session closed for user paleksicFeb  1 05:02:38 g-k8s-master-0 sshd[653]: Accepted publickey for paleksic from 10.10.40.2 port 47946 ssh2: RSA SHA256:rHk7JkW44vqcomA66hsgmX14CAtCE1KytT3/lc4CoDoFeb  1 05:02:38 g-k8s-master-0 sshd[653]: pam_unix(sshd:session): session opened for user paleksic by (uid=0)Feb  1 05:02:38 g-k8s-master-0 systemd-logind[1025]: New session 1156 of user paleksic.Feb  1 05:02:38 g-k8s-master-0 systemd: pam_unix(systemd-user:session): session opened for user paleksic by (uid=0)"}

I guess I don't know how to add new sources properly. Here is the complete configuration

# This makes it easier for users to search for logs by pod name or by
   # the name of the Kubernetes container regardless of how many times the
   # Kubernetes pod has been restarted (resulting in a several Docker container IDs).
   # Json Log Example:
   # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
   # CRI Log Example:
   # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here
   <source>
     @id fluentd-containers.log
     @type tail
     path /var/log/containers/*.log
     pos_file /var/log/es-containers.log.pos
     tag raw.kubernetes.*
     read_from_head true
     <parse>
       @type multi_format
       <pattern>
         format json
         time_key time
         time_format %Y-%m-%dT%H:%M:%S.%NZ
       </pattern>
       <pattern>
         format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
         time_format %Y-%m-%dT%H:%M:%S.%N%:z
       </pattern>
     </parse>
   </source>
   # Detect exceptions in the log output and forward them as one log entry.
   <match raw.kubernetes.**>
     @id raw.kubernetes
     @type detect_exceptions
     remove_tag_prefix raw
     message log
     stream stream
     multiline_flush_interval 5
     max_bytes 500000
     max_lines 1000
   </match>
 system.input.conf: |-
   # Example:
   # 2015-12-21 23:17:22,066 [salt.state       ][INFO    ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081
   <source>
     @id minion
     @type tail
     format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
     time_format %Y-%m-%d %H:%M:%S
     path /var/log/salt/minion
     pos_file /var/log/salt.pos
     tag salt
   </source>
   # Example:
   # Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script
   <source>
     @id startupscript.log
     @type tail
     format syslog
     path /var/log/startupscript.log
     pos_file /var/log/es-startupscript.log.pos
     tag startupscript
   </source>
   # Examples:
   # time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
   # time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
   # TODO(random-liu): Remove this after cri container runtime rolls out.
   <source>
     @id docker.log
     @type tail
     format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
     path /var/log/docker.log
     pos_file /var/log/es-docker.log.pos
     tag docker
   </source>
   # Example:
   # 2016/02/04 06:52:38 filePurge: successfully removed file /var/etcd/data/member/wal/00000000000006d0-00000000010a23d1.wal
   <source>
     @id etcd.log
     @type tail
     # Not parsing this, because it doesn't have anything particularly useful to
     # parse out of it (like severities).
     format none
     path /var/log/etcd.log
     pos_file /var/log/es-etcd.log.pos
     tag etcd
   </source>
   # Multi-line parsing is required for all the kube logs because very large log
   # statements, such as those that include entire object bodies, get split into
   # multiple lines by glog.
   # Example:
   # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
   <source>
     @id kubelet.log
     @type tail
     format multiline
     multiline_flush_interval 5s
     format_firstline /^\w\d{4}/
     format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
     time_format %m%d %H:%M:%S.%N
     path /var/log/kubelet.log
     pos_file /var/log/es-kubelet.log.pos
     tag kubelet
   </source>
   # Example:
   # I1118 21:26:53.975789       6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
   <source>
     @id authlog
     @type tail
     format none
     path /var/log/auth.log
     pos_file /var/log/auth.pos
     tag authlog
   </source>
   <match authlog>
     @type stdout
   </match>
   <source>
     @id kube-proxy.log
     @type tail
     format multiline
     multiline_flush_interval 5s
     format_firstline /^\w\d{4}/
     format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
     time_format %m%d %H:%M:%S.%N
     path /var/log/kube-proxy.log
     pos_file /var/log/es-kube-proxy.log.pos
     tag kube-proxy
   </source>
   # Example:
   # I0204 07:00:19.604280       5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
   <source>
     @id kube-apiserver.log
     @type tail
     format multiline
     multiline_flush_interval 5s
     format_firstline /^\w\d{4}/
     format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
     time_format %m%d %H:%M:%S.%N
     path /var/log/kube-apiserver.log
     pos_file /var/log/es-kube-apiserver.log.pos
     tag kube-apiserver
   </source>
   # Example:
   # I0204 06:55:31.872680       5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
   <source>
     @id kube-controller-manager.log
     @type tail
     format multiline
     multiline_flush_interval 5s
     format_firstline /^\w\d{4}/
     format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
     time_format %m%d %H:%M:%S.%N
     path /var/log/kube-controller-manager.log
     pos_file /var/log/es-kube-controller-manager.log.pos
     tag kube-controller-manager
   </source>
   # Example:
   # W0204 06:49:18.239674       7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
   <source>
     @id kube-scheduler.log
     @type tail
     format multiline
     multiline_flush_interval 5s
     format_firstline /^\w\d{4}/
     format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
     time_format %m%d %H:%M:%S.%N
     path /var/log/kube-scheduler.log
     pos_file /var/log/es-kube-scheduler.log.pos
     tag kube-scheduler
   </source>
   # Example:
   # I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
   <source>
     @id glbc.log
     @type tail
     format multiline
     multiline_flush_interval 5s
     format_firstline /^\w\d{4}/
     format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
     time_format %m%d %H:%M:%S.%N
     path /var/log/glbc.log
     pos_file /var/log/es-glbc.log.pos
     tag glbc
   </source>
   # Example:
   # I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
   <source>
     @id cluster-autoscaler.log
     @type tail
     format multiline
     multiline_flush_interval 5s
     format_firstline /^\w\d{4}/
     format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
     time_format %m%d %H:%M:%S.%N
     path /var/log/cluster-autoscaler.log
     pos_file /var/log/es-cluster-autoscaler.log.pos
     tag cluster-autoscaler
   </source>
   # Logs from systemd-journal for interesting services.
   # TODO(random-liu): Remove this after cri container runtime rolls out.
   <source>
     @id journald-docker
     @type systemd
     matches [{ "_SYSTEMD_UNIT": "docker.service" }]
     <storage>
       @type local
       persistent true
       path /var/log/journald-docker.pos
     </storage>
     read_from_head true
     tag docker
   </source>
   <source>
     @id journald-container-runtime
     @type systemd
     matches [{ "_SYSTEMD_UNIT": "{{ fluentd_container_runtime_service }}.service" }]
     <storage>
       @type local
       persistent true
       path /var/log/journald-container-runtime.pos
     </storage>
     read_from_head true
     tag container-runtime
   </source>
   <source>
     @id journald-kubelet
     @type systemd
     matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
     <storage>
       @type local
       persistent true
       path /var/log/journald-kubelet.pos
     </storage>
     read_from_head true
     tag kubelet
   </source>
   <source>
     @id journald-node-problem-detector
     @type systemd
     matches [{ "_SYSTEMD_UNIT": "node-problem-detector.service" }]
     <storage>
       @type local
       persistent true
       path /var/log/journald-node-problem-detector.pos
     </storage>
     read_from_head true
     tag node-problem-detector
   </source>
   <source>
     @id kernel
     @type systemd
     matches [{ "_TRANSPORT": "kernel" }]
     <storage>
       @type local
       persistent true
       path /var/log/kernel.pos
     </storage>
     <entry>
       fields_strip_underscores true
       fields_lowercase true
     </entry>
     read_from_head true
     tag kernel
   </source>
 forward.input.conf: |-
   # Takes the messages sent over TCP
   <source>
     @type forward
   </source>
 monitoring.conf: |-
   # Prometheus Exporter Plugin
   # input plugin that exports metrics
   <source>
     @type prometheus
   </source>
   <source>
     @type monitor_agent
   </source>
   # input plugin that collects metrics from MonitorAgent
   <source>
     @type prometheus_monitor
     <labels>
       host ${hostname}
     </labels>
   </source>
   # input plugin that collects metrics for output plugin
   <source>
     @type prometheus_output_monitor
     <labels>
       host ${hostname}
     </labels>
   </source>
   # input plugin that collects metrics for in_tail plugin
   <source>
     @type prometheus_tail_monitor
     <labels>
       host ${hostname}
     </labels>
   </source>
 output.conf: |-
   # Enriches records with Kubernetes metadata
   <filter kubernetes.**>
     @type kubernetes_metadata
   </filter>
   # Concatenate multi-line logs
   <filter **>
     @type concat
     key message
     multiline_end_regexp /\n$/
     separator ""
   </filter>
   <match **>
     @id elasticsearch
     @type elasticsearch
     @log_level info
     type_name fluentd
     include_tag_key true
     host elasticsearch-logging
     port 9200
     logstash_format true
     <buffer>
       @type file
       path /var/log/fluentd-buffers/kubernetes.system.buffer
       flush_mode interval
       retry_type exponential_backoff
       flush_thread_count 2
       flush_interval 5s
       retry_forever
       retry_max_interval 30
       chunk_limit_size 2M
       queue_limit_length 8
       overflow_action block
     </buffer>
   </match>

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.