K8S 部署ELK日志采集系统——筑梦之路

部署elasticsearch
#定义pvc
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: elasticsearch-pvc-data
  namespace: merry
spec:
  accessModes:
     - ReadWriteOnce
  volumeMode: Filesystem
  storageClassName: merry-data-managed-nfs-storage
  resources:
     requests:
       storage: 1000Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: elasticsearch-pvc-log
  namespace: merry
spec:
  accessModes:
     - ReadWriteOnce
  volumeMode: Filesystem
  storageClassName: merry-log-managed-nfs-storage
  resources:
     requests:
       storage: 1000Gi

#定义cm
apiVersion: v1
kind: ConfigMap
metadata:
  name: elasticsearch-cm
  namespace: merry
data:
  elasticsearch.yml: |-
    cluster.name: "docker-cluster"
    network.host: 0.0.0.0
    xpack.security.enabled: true
    http.cors.allow-headers: Authorization
    xpack.security.transport.ssl.enabled: true

#定义deploy
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    run: elasticsearch
  name: elasticsearch-deploy
  namespace: merry
spec:
  replicas: 1
  selector:
    matchLabels:
      run: elasticsearch
  template:
    metadata:
      labels:
        run: elasticsearch
    spec:
      volumes:
      - name: elasticsearch-data
        persistentVolumeClaim:
          claimName: elasticsearch-pvc-data
      - name: elasticsearch-log
        persistentVolumeClaim:
          claimName: elasticsearch-pvc-log
      - name: elasticsearch-cm
        configMap:
          name: elasticsearch-cm
      containers:
      - image: elasticsearch:7.4.0-ik
        imagePullPolicy: IfNotPresent
        name: elasticsearch
        env:
        - name: discovery.type
          value: "single-node"
        volumeMounts:
        - mountPath: "/usr/share/elasticsearch/data"
          name: elasticsearch-data
        - mountPath: "/usr/share/elasticsearch/logs"
          name: elasticsearch-log
        - mountPath: "/usr/share/elasticsearch/config/elasticsearch.yml"
          name: elasticsearch-cm
          subPath: elasticsearch.yml
        ports:
          - name: http9200
            containerPort: 9200
            protocol: TCP
          - name: http9300
            containerPort: 9300
            protocol: TCP
        livenessProbe:
            tcpSocket:
              port: http9200
            initialDelaySeconds: 15
            periodSeconds: 20
          readinessProbe:
            httpGet:
              path: /_cat/health
              port: http9200
            initialDelaySeconds: 5
            periodSeconds: 10
        resources:
          limits: 
            cpu: 2000m
            memory: 4096Mi
          requests: 
            cpu: 20m
            memory: 1024Mi
      restartPolicy: Always

#定义svc
apiVersion: v1
kind: Service
metadata:
  labels:
    run: elasticsearch
  name: elasticsearch-svc
  namespace: merry
spec:
  ports:
  - port: 9200
    protocol: TCP
    targetPort: 9200
    nodePort: 39200
    name: p9200
  - port: 9300
    protocol: TCP
    targetPort: 9300
    name: p9300
  selector:
    run: elasticsearch
  type: NodePort

部署kibana
#定义cm
apiVersion: v1
kind: ConfigMap
metadata:
  name: kibana-cm
  namespace: merry
data:
  kibana.yml: |-
    server.name: kibana
    server.host: "0"
    elasticsearch.hosts: [ "http://elasticsearch-svc:9200" ]
    elasticsearch.username: "elastic"
    elasticsearch.password: "admin12345678"

#定义deploy
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    elastic-app: kibana
  name: kibana
  namespace: merry
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      elastic-app: kibana
  template:
    metadata:
      labels:
        elastic-app: kibana
    spec:
      restartPolicy: Always
      volumes:
        - name: kibana-yml
          configMap:
            name: kibana-cm
      containers:
        - name: kibana
          image: kibana:7.4.0
          ports:
            - containerPort: 5601
              protocol: TCP
          volumeMounts:
            - mountPath: "/usr/share/kibana/config/kibana.yml"
              name: kibana-yml
              subPath: kibana.yml
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule

#定义svc
kind: Service
apiVersion: v1
metadata:
  labels:
    elastic-app: kibana
  name: kibana-svc
  namespace: merry
spec:
  ports:
    - port: 5601
      targetPort: 5601
      nodePort: 35601
  selector:
    elastic-app: kibana
  type: NodePort
---
部署logstash
#定义cm
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-conf-cm
  namespace: merry
data:
  logstash.conf: |-
    input{
       beats{
             port => 5044
     }
    
  
        filter{
            #需要配置否则host是一个json不是文本则无法输出至elasticsearch
            mutate {
                rename => { "[host][name]" => "host" }
           }
    }
  
    output{
        elasticsearch{
            hosts => ["elasticsearch-svc:9200"]
            index => "logstash-%{+YYYY.MM.dd}"
        }

        stdout{
            codec => rubydebug
        }
    }
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-yml-cm
  namespace: merry
data:
  logstash.yml: |-
    http.host: "0.0.0.0"
    xpack.monitoring.elasticsearch.url: http://elasticsearch-svc:9200

---
#定义deploy
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app: logstash
  name: logstash
  namespace: merry
spec:
  serviceName: "logstash"
  replicas: 1
  selector:
    matchLabels:
      app: logstash
  template:
    metadata:
      labels:
        app: logstash
    spec:
      containers:
      - image: logstash:7.4.0
        name: logstash
        volumeMounts:
        - name: logstash-yml-config
          mountPath: /usr/share/logstash/config/logstash.yml
          subPath: logstash.yml
        - name: logstash-config
          mountPath: /usr/share/logstash/pipeline/logstash.conf
          subPath: logstash.conf
      volumes:
      - name: logstash-yml-config
        configMap:
          name: logstash-yml-cm
      - name: logstash-config
        configMap:
          name: logstash-conf-cm
---
#定义svc
kind: Service
apiVersion: v1
metadata:
  labels:
    app: logstash
  name: logstash-svc
  namespace: merry
spec:
  ports:
    - port: 5044
      targetPort: 5044
      nodePort: 35044
  selector:
    app: logstash
  type: NodePort

---

部署filebeat
#filebeat-cm.yml
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-cm
  namespace: merry
data:
  filebeat.yml: |-
    filebeat.inputs:
    - type: log
      enabled: true
      paths:
        #容器日志路径为/messages
        - /messages
      fields:
        app: k8s
        type: module
    filebeat.config.modules:
      path: ${path.config}/modules.d/*.yml
      reload.enabled: false
    setup.template.settings:
      index.number_of_shards: 3
    setup.kibana:
    output.logstash:
      hosts: ["logstash-svc:5044"]
    processors:
      - add_host_metadata:
      - add_cloud_metadata:
---
#filebeat-deploy.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    app: filebeat
  name: filebeat
  namespace: merry
spec:
  selector:
    matchLabels:
      app: filebeat
  template:
    metadata:
      labels:
        app: filebeat
    spec:
      containers:
      - image: filebeat:7.4.0
        name: filebeat
        volumeMounts:
        - name: filebeat-config
          mountPath: /etc/filebeat.yml
          subPath: filebeat.yml
        - name: k8s-system-logs
          mountPath: /messages
        #使用配置文件启动filebeat
        args: [
          "-c", "/etc/filebeat.yml",
          "-e",
        ]
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            cpu: 500m
            memory: 500Mi
        #设置访问容器的用户ID本次设置为0即访问容器为root用户
        #不设置默认容器用户为filebeat则会出现访问日志文件没权限的问题
        #设置该参数使用kubelet exec登录容器的用户为root用户
        securityContext:
          runAsUser: 0
      volumes:
      - name: filebeat-config
        configMap:
          name: filebeat-cm
      #把主机的日志/var/logs/messages挂载至容器
      - name: k8s-system-logs
        hostPath:
          path: /var/log/messages
          type: File

---
以上内容仅供学习参考