filebeat kubernetes logger to ship logs to logstash filter running on host machine (10.0.2.2)

kubernetes-autodiscover-logstash.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: kube-system
  labels:
    k8s-app: filebeat
data:
  filebeat.yml: |-
    filebeat.autodiscover:
      providers:
        - type: kubernetes
          templates:
            - condition:
                or:
                  - equals:
                      kubernetes.namespace: kube-system
                  - equals:
                      kubernetes.namespace: default
              config:
                - type: docker
                  hints.enabled: true
                  containers.ids:
                    - "${data.kubernetes.container.id}"
                  # First option to pass metadata
                  fields:
                    _service: "${data.kubernetes.labels.ut-service}"
                    _token: "${data.kubernetes.labels.ut-token}"
                    type: "${data.kubernetes.labels.ut-type}"
                  fields_under_root: true
                  exclude_lines: ["^\\s+[\\-`('.|_]"]  # drop asciiart lines
                  multiline.pattern: '^[[:space:]]'
                  multiline.negate: false
                  multiline.match: after
    processors:
      - add_kubernetes_metadata:
          in_cluster: true

      # This ensures that every log that passes has required fields.
      #- drop_event.when.not:
      #    has_fields: ['kubernetes.labels.ut-service', 'kubernetes.labels.ut-token', 'kubernetes.labels.ut-type']
      
      # Second option to pass metadata, this renames fields to outer most json level. Info is moved from kubernetes.labels
      #- rename:
      #    fields:
      #      - from: "kubernetes.labels.ut-service"
      #        to: "_service"
      #      - from: "kubernetes.labels.ut-token"
      #        to: "_token"
      #      - from: "kubernetes.labels.ut-type"
      #        to: "type"
    output.logstash:
      hosts: ['${LOGSTASH_HOSTS}']


---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-inputs
  namespace: kube-system
  labels:
    k8s-app: filebeat
data:
  kubernetes.yml: |-
    - type: docker
      containers.ids:
      - "*"
      processors:
        - add_kubernetes_metadata:
            in_cluster: true
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: filebeat
  namespace: kube-system
  labels:
    k8s-app: filebeat
    ut-service: ship
    ut-type: log
    ut-token: herro
spec:
  template:
    metadata:
      labels:
        k8s-app: filebeat
        ut-service: ship
        ut-type: log
        ut-token: 123abc321asd
    spec:
      serviceAccountName: filebeat
      terminationGracePeriodSeconds: 30
      containers:
      - name: filebeat
        image: docker.elastic.co/beats/filebeat:6.3.2
        args: [
          "-c",
          "/etc/filebeat.yml",
          "-e",
        ]
        env:
        - name: LOGSTASH_HOSTS
          value: 10.0.2.2:5044
        - name: LOG_LEVEL
          value: debug
        - name: CLUSTER_NAME
          value: my_cluster
        - name: FILEBEAT_HOST
          valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
        securityContext:
          runAsUser: 0
        resources:
          limits:
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        volumeMounts:
        - name: config
          mountPath: /etc/filebeat.yml
          readOnly: true
          subPath: filebeat.yml
        - name: inputs
          mountPath: /usr/share/filebeat/inputs.d
          readOnly: true
        - name: data
          mountPath: /usr/share/filebeat/data
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: dockersock
          mountPath: /var/run/docker.sock
      volumes:
      - name: config
        configMap:
          defaultMode: 0600
          name: filebeat-config
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: inputs
        configMap:
          defaultMode: 0600
          name: filebeat-inputs
      # We set an `emptyDir` here to ensure the manifest will deploy correctly.
      # It's recommended to change this to a `hostPath` folder, to ensure internal data
      # files survive pod changes (ie: version upgrade)
      - name: data
        emptyDir: {}
      - name: dockersock
        hostPath:
          path: /var/run/docker.sock
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: filebeat
subjects:
- kind: ServiceAccount
  name: filebeat
  namespace: kube-system
roleRef:
  kind: ClusterRole
  name: filebeat
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: filebeat
  labels:
    k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
  resources:
  - namespaces
  - pods
  verbs:
  - get
  - watch
  - list
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat
  namespace: kube-system
  labels:
    k8s-app: filebeat
---