kubernetes监控:grafana plugins IN kubernetes
说明：使用grafana官方plugins(grafana-kuberneres-app)链接prometheus展示kubernetes的信息注意：所有与监控有关的信息都放在namespace=monitor上
github:https://github.com/grafana/kubernetes-app
１、安装prometheus
prom-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: prometheus
namespace: monitor
rules:
- apiGroups: [""]
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
- apiGroups:
- extensions
resources:
- ingresses
verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: monitor
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: monitor
cat prom-config-kubernetes.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-config
namespace: monitor
data:
prometheus.yml: |
global:
scrape_configs:
- job_name: 'kubernetes-kubelet'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc.cluster.local:443
- source_labels:
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics
- job_name: 'kubernetes-cadvisor'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc.cluster.local:443
- source_labels:
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-kube-state'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels:
action: replace
target_label: kubernetes_namespace
- source_labels:
action: replace
target_label: kubernetes_pod_name
- source_labels:
regex: .*true.*
action: keep
- source_labels: ['__meta_kubernetes_pod_label_daemon', '__meta_kubernetes_pod_node_name']
regex: 'node-exporter;(.*)'
action: replace
target_label: nodename
cat prom-install.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-config-test
namespace: monitor
data:
prometheus.yml: |
global:
scrape_configs:
- job_name: 'kubernetes-kubelet'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc.cluster.local:443
- source_labels:
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics
- job_name: 'kubernetes-cadvisor'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc.cluster.local:443
- source_labels:
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-kube-state'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels:
action: replace
target_label: kubernetes_namespace
- source_labels:
action: replace
target_label: kubernetes_pod_name
- source_labels:
regex: .*true.*
action: keep
- source_labels: ['__meta_kubernetes_pod_label_daemon', '__meta_kubernetes_pod_node_name']
regex: 'node-exporter;(.*)'
action: replace
target_label: nodename
- job_name: 'tomcat-pods'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels:
regex: tomcat
action: keep
- source_labels:
regex: 1234
action: keep
- source_labels:
action: replace
target_label: kubernetes_pod_name
# cat prom/prom-install.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
name: prometheus
name: prometheus
namespace: monitor
spec:
replicas: 1
template:
metadata:
labels:
app: prometheus-server
spec:
serviceAccountName: prometheus
containers:
- name: prometheus
image: hub.llzg.com.cn/monitor/prometheus:latest
imagePullPolicy: Always
ports:
- containerPort: 9090
protocol: TCP
volumeMounts:
- mountPath: "/etc/prometheus"
name: config-prometheus
imagePullSecrets:
- name: authllzg
volumes:
- name: config-prometheus
configMap:
name: prometheus-config
cat prom-svc.yaml
kind: Service
apiVersion: v1
metadata:
name: prometheus-ingress-service
namespace: monitor
spec:
selector:
app: prometheus-server
ports:
- protocol: TCP
port: 9090
name: prom
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: prom-web-ui
namespace: monitor
spec:
rules:
- host: prom.name.com.cn
http:
paths:
- path: /
backend:
serviceName: prometheus-ingress-service
servicePort: prom
2、安装state-metrics
cat state-metrics-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-state-metrics
namespace: monitor
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
namespace: monitor
name: kube-state-metrics-resizer
rules:
- apiGroups: [""]
resources:
- pods
verbs: ["get"]
- apiGroups: ["extensions"]
resources:
- deployments
resourceNames: ["kube-state-metrics"]
verbs: ["get", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kube-state-metrics
namespace: monitor
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kube-state-metrics-resizer
subjects:
- kind: ServiceAccount
name: kube-state-metrics
namespace: monitor
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kube-state-metrics
namespace: monitor
rules:
- apiGroups: [""]
resources:
- nodes
- pods
- services
- resourcequotas
- replicationcontrollers
- limitranges
- persistentvolumeclaims
- persistentvolumes
- namespaces
- endpoints
verbs: ["list", "watch"]
- apiGroups: ["extensions"]
resources:
- daemonsets
- deployments
- replicasets
verbs: ["list", "watch"]
- apiGroups: ["apps"]
resources:
- statefulsets
verbs: ["list", "watch"]
- apiGroups: ["batch"]
resources:
- cronjobs
- jobs
verbs: ["list", "watch"]
- apiGroups: ["autoscaling"]
resources:
- horizontalpodautoscalers
verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kube-state-metrics
namespace: monitor
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kube-state-metrics
subjects:
- kind: ServiceAccount
name: kube-state-metrics
namespace: monitor
---
{
"apiVersion": "apps/v1beta1",
"kind": "Deployment",
"metadata": {
"name": "kube-state-metrics",
"namespace": "monitor"
},
"spec": {
"selector": {
"matchLabels": {
"k8s-app": "kube-state-metrics",
"grafanak8sapp": "true"
}
},
"replicas": 1,
"template": {
"metadata": {
"labels": {
"k8s-app": "kube-state-metrics",
"grafanak8sapp": "true"
}
},
"spec": {
"serviceAccountName": "kube-state-metrics",
"containers": [
{
"name": "kube-state-metrics",
"image": "hub.llzg.com.cn/monitor/kube-state-metrics:v1.1.0",
"ports": [
{
"name": "http-metrics",
"containerPort": 8080
}
],
"readinessProbe": {
"httpGet": {
"path": "/healthz",
"port": 8080
},
"initialDelaySeconds": 5,
"timeoutSeconds": 5
}
}
],
"imagePullSecrets": [
{
"name": "authllzg"
}
]
}
}
}
}
3、安装node-exporter
cat node-exporter.yaml
{
"kind": "DaemonSet",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "node-exporter",
"namespace": "monitor"
},
"spec": {
"selector": {
"matchLabels": {
"daemon": "node-exporter",
"grafanak8sapp": "true"
}
},
"template": {
"metadata": {
"name": "node-exporter",
"labels": {
"daemon": "node-exporter",
"grafanak8sapp": "true"
}
},
"spec": {
"volumes": [
{
"name": "proc",
"hostPath": {
"path": "/proc"
}
},
{
"name": "sys",
"hostPath": {
"path": "/sys"
}
}
],
"containers": [
{
"name": "node-exporter",
"image": "hub.llzg.com.cn/monitor/node-exporter:v0.15.0",
"args": [
"--path.procfs=/proc_host",
"--path.sysfs=/host_sys"
],
"ports": [
{
"name": "node-exporter",
"hostPort": 9100,
"containerPort": 9100
}
],
"volumeMounts": [
{
"name": "sys",
"readOnly": true,
"mountPath": "/host_sys"
},
{
"name": "proc",
"readOnly": true,
"mountPath": "/proc_host"
}
],
"imagePullPolicy": "IfNotPresent"
}
],
"restartPolicy": "Always",
"hostNetwork": true,
"hostPID": true
}
}
}
}
4、安装grafana(添加grafana-kubernetes-app插件)
cat grafana.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
name: grafana-server
name: grafana
namespace: monitor
spec:
replicas: 1
template:
metadata:
labels:
app: grafana-server
spec:
serviceAccountName: prometheus
containers:
- name: grafana
image: hub.llzg.com.cn/monitor/grafana:latest
ports:
- containerPort: 3000
protocol: TCP
volumeMounts:
- mountPath: "/var/lib/grafana"
readOnly: false
name: grafana-nfs
env:
- name: GF_INSTALL_PLUGINS
value: "grafana-kubernetes-app"
imagePullSecrets:
- name: authllzg
volumes:
- name: grafana-nfs
nfs:
server: 192.168.10.28
path: "/data/nfs/monitor/grafana"
---
kind: Service
apiVersion: v1
metadata:
name: grafana-ingress-service
namespace: monitor
spec:
selector:
app: grafana-server
ports:
- protocol: TCP
port: 3000
name: grafana
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: grafana-ingress
namespace: monitor
annotations:
kubernetes.io/ingress.class: traefik
spec:
rules:
- host: grafana.llzg.com.cn
http:
paths:
- path: /
backend:
serviceName: grafana-ingress-service
servicePort: grafana
５、grafana+prometheus监控显示kubernetes
(1)grafana添加prometheus
(2)加载grafana-kubernetes-app插件
configuration---->plugins--------->kubernetes
(3)配置grafana-kubernetes-app插件
４、OK，去看图吧
页:
[1]