部署prometheus、grafana、alertmanager

groups:   - name: Node-rules     rules:     - alert: Node-Down       expr: up{job="node1"} == 0       for: 1m       labels:         severity: 严重警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{$labels.instance }} 节点已经宕机 1分钟"         description: "节点宕机"      - alert: Node-CpuHigh       expr: (1 - avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="idle"}[5m]))) * 100 > 80       for: 1m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} cpu使用率超 80%"         description: "CPU 使用率为 {{ $value }}%"      - alert: Node-CpuIowaitHigh       expr: avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="iowait"}[5m])) * 100 > 80       for: 1m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} CPU iowait 使用率超过 80%"         description: "CPU iowait 使用率为 {{ $value }}%"      - alert: Node-MemoryHigh       expr: (1 - node_memory_MemAvailable_bytes{job="node"} / node_memory_MemTotal_bytes{job="node"}) * 100 > 80       for: 1m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Memory使用率超过 80%"         description: "Memory 使用率为 {{ $value }}%"      - alert: Node-Load5High       expr: node_load5 > (count by (instance) (node_cpu_seconds_total{job="node",mode='system'})) * 1.2       for: 1m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Load(5m)过高,超出cpu核数1.2倍"         description: "Load(5m)过高,超出cpu核数 1.2倍"      - alert: Node-DiskRootHigh       expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"}) * 100 > 80       for: 10m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Disk(/ 分区) 使用率超过 80%"         description: "Disk(/ 分区) 使用率为 {{ $value }}%"      - alert: Node-DiskDataHigh       expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/data"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/data"}) * 100 > 80       for: 10m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Disk(/data 分区) 使用率超过 80%"         description: "Disk(/data 分区) 使用率为 {{ $value }}%"      - alert: Node-DiskReadHigh       expr: irate(node_disk_read_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)       for: 1m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Disk 读取字节数速率超过 20 MB/s"         description: "Disk 读取字节数速率为 {{ $value }}MB/s"      - alert: Node-DiskWriteHigh       expr: irate(node_disk_written_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)       for: 1m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Disk 写入字节数速率超过 20 MB/s"         description: "Disk 写入字节数速率为 {{ $value }}MB/s"      - alert: Node-DiskReadRateCountHigh       expr: irate(node_disk_reads_completed_total{job="node"}[5m]) > 3000       for: 1m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Disk iops 每秒读取速率超过 3000 iops"         description: "Disk iops 每秒读取速率为 {{ $value }}"      - alert: Node-DiskWriteRateCountHigh       expr: irate(node_disk_writes_completed_total{job="node"}[5m]) > 3000       for: 1m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Disk iops 每秒写入速率超过 3000 iops"         description: "Disk iops 每秒写入速率为 {{ $value }}"      - alert: Node-InodeRootUsedPercentHigh       expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/"}) * 100 > 80       for: 10m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Disk (/ 分区) inode 使用率超过 80%"         description: "Disk (/ 分区) inode 使用率为 {{ $value }}%"      - alert: Node-InodeBootUsedPercentHigh       expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/data"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/data"}) * 100 > 80       for: 10m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Disk (/data 分区) inode 使用率超过 80%"         description: "Disk (/data 分区) inode 使用率为 {{ $value }}%"      - alert: Node-FilefdAllocatedPercentHigh       expr: node_filefd_allocated{job="node"} / node_filefd_maximum{job="node"} * 100 > 80       for: 10m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Filefd 打开百分比超过 80%"         description: "Filefd 打开百分比为 {{ $value }}%"      - alert: Node-NetworkNetinBitRateHigh       expr: avg by (instance) (irate(node_network_receive_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8       for: 3m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Network 接收比特数速率超过 20MB/s"         description: "Network 接收比特数速率为 {{ $value }}MB/s"      - alert: Node-NetworkNetoutBitRateHigh       expr: avg by (instance) (irate(node_network_transmit_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8       for: 3m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Network 接收比特数速率超过 20MB/s"         description: "Network 发送比特数速率为 {{ $value }}MB/s"      - alert: Node-NetworkNetinPacketErrorRateHigh       expr: avg by (instance) (irate(node_network_receive_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15       for: 3m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Network 接收错误包速率超过 15个/秒"         description: "Network 接收错误包速率为 {{ $value }}个/秒"      - alert: Node-NetworkNetoutPacketErrorRateHigh       expr: avg by (instance) (irate(node_network_transmit_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15       for: 3m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Network 发送错误包速率超过 15个/秒"         description: "Network 发送错误包速率为 {{ $value }}个/秒"      - alert: Node-ProcessBlockedHigh       expr: node_procs_blocked{job="node"} > 10       for: 10m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} Process 当前被阻塞的任务的数量超过 10个"         description: "Process 当前被阻塞的任务的数量为 {{ $value }}个"      - alert: Node-TimeOffsetHigh       expr: abs(node_timex_offset_seconds{job="node"}) > 3 * 60       for: 2m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} 节点的时间偏差超过 3m"         description: "节点的时间偏差为 {{ $value }}m"      - alert: Node-TCPconnection       expr: node_sockstat_TCP_tw{job="node"} > 15000       for: 2m       labels:         severity: 警告         instance: "{{ $labels.instance }}"       annotations:         summary: "{{ $labels.instance }} TCP 等待关闭的TCP连接数TIME_WAIT过高大于15000"         description: "TCP 等待关闭的TCP连接数为 {{ $value }}"

8. 配置alertmanager邮件报警