Master Filebeat: Complete Configuration Guide for Log Shipping

This guide presents a full Filebeat configuration for collecting and shipping logs to Elasticsearch, including input definitions, field enrichment, multiline patterns, module setup, index template settings, authentication, and processors to drop unnecessary fields.

# configuration file.
#======Filebeat inputs ======
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /mnt/logfile-nas/crs-center-service-*/*.log
  fields:
    app_name: appname
    profiles_active: pro
    app_node: hostname
# fields are added under the root
fields_under_root: true
multiline.type: pattern
multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
multiline.negate: true
multiline.match: after
# ====== Filebeat modules ======
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
# ====== Elasticsearch template setting =======
setup.template.name: "template"
setup.template.pattern: "appname-*"
setup.template.settings.index.number_of_shards: 3
setup.template.settings.index.number_of_replicas: 1
# Disable index lifecycle management; if enabled it would ignore our custom index
setup.ilm.enabled: false
setup.kibana:
# ======
# Elasticsearch Output
# =======
output.elasticsearch:
  hosts: ["es1:9200", "es2:9200", "es3:9200"]
  index: "%{[app_name]}-%{[profiles_active]}-%{+YYYY.MM.dd}"
  username: "username"
  password: "password"
# ====== Processors ======
processors:
- drop_fields:
    fields: ["ecs","host","agent","log","input"]