EMQX5集群搭建

系统优化

建议执行完全部的优化策略后重新启动服务器。

1、关闭swap分区

swapoff -a && sed -i '/swap/d' /etc/fstab

2、系统全局允许分配的最大文件句柄数

echo "fs.file-max = 2097152" >> /etc/sysctl.conf && \
echo "fs.nr_open = 2097152" >> /etc/sysctl.conf
sysctl -p

3、网络及系统优化设置

# 并发连接backlog的优化设置
echo "net.core.somaxconn = 32768" >>  /etc/sysctl.conf && \
echo "net.ipv4.tcp_max_syn_backlog = 16384" >>  /etc/sysctl.conf && \
echo "net.core.netdev_max_backlog = 16384" >>  /etc/sysctl.conf

# 可暴露的端口范围
echo "net.ipv4.ip_local_port_range='1024 65535'" >>  /etc/sysctl.conf 

# Socket的读写缓存优化
echo "net.core.rmem_default = 262144" >>  /etc/sysctl.conf && \
echo "net.core.wmem_default = 262144" >>  /etc/sysctl.conf && \
echo "net.core.rmem_max = 16777216" >>  /etc/sysctl.conf && \
echo "net.core.wmem_max = 16777216" >>  /etc/sysctl.conf && \
echo "net.core.optmem_max = 16777216" >>  /etc/sysctl.conf

echo "net.ipv4.tcp_rmem='1024 4096 16777216'" >>  /etc/sysctl.conf&& \
echo "net.ipv4.tcp_wmem='1024 4096 16777216'" >>  /etc/sysctl.conf

# Tcp连接追踪
echo "net.nf_conntrack_max = 1000000" >>  /etc/sysctl.conf&& \
echo "net.netfilter.nf_conntrack_max = 1000000" >>  /etc/sysctl.conf
echo "net.netfilter.nf_conntrack_tcp_timeout_time_wait = 30" >>  /etc/sysctl.conf

# TIME-WAIT Socket 最大数量、回收与重用设置
echo "net.ipv4.tcp_max_tw_buckets = 1048576" >>  /etc/sysctl.conf

# FIN-WAIT-2 Socket 超时设置
echo "net.ipv4.tcp_fin_timeout = 15" >>  /etc/sysctl.conf
  • somaxconn：这是Linux Kernel中的一个参数，用于设置socket监听（Listen）的backlog上限。backlog就是Socket的监听队列，当一个请求尚未被处理或建立时，它会进入backlog。socket server处理请求后会，这些请求就不再位于监听队列中了。当server处理请求较慢时，直至backlog被请求填满时，新来的请求就会被拒绝。

配置集群

1、设置主机名

127.0.0.1 localhost
127.0.1.1 emqx1.aolingo.com

# The following lines are desirable for IPv6 capable hosts
::1     ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters

192.168.50.27   emqx1.aolingo.com
192.168.50.28   emqx2.aolingo.com
192.168.50.29   emqx3.aolingo.com

2、生成根证书

openssl genrsa -out ca.key 2048
openssl req -x509 -new -nodes -key ca.key -sha256 -days 3650 -out ca.pem -subj "/C=CN/ST=BeiJing/L=Beijing/O=dokbok/OU=dokbok"

3、创建ssl配置文件

#vim ssl.cnf
[req]
default_bits  = 2048
distinguished_name = req_distinguished_name
req_extensions = req_ext
x509_extensions = v3_req
prompt = no

#组织机构信息，不要与上面的创建ca证书的时候一致
[req_distinguished_name]
countryName = CN
stateOrProvinceName = BeiJing
localityName = BeiJing
organizationName = aolingo
commonName = aolingo

[req_ext]
subjectAltName = @alt_names

[v3_req]
subjectAltName = @alt_names

[alt_names]
IP.1 = 192.168.2.21
IP.2 = 192.168.2.22
IP.3 = 192.168.2.23

DNS.1 = 127.0.0.1
DNS.2 = localhost

4、生成服务器证书

注意：Server 证书的组织机构信息不能与 CA 机构的一样，否则无法通过验证

openssl genrsa -out server-key.pem 2048
openssl req -new -key ./server-key.pem -config ssl.cnf -out server-csr.pem
openssl x509 -req -in ./server-csr.pem -CA ca.pem -CAkey ca.key -CAcreateserial -out server-cert.pem -days 3650 -sha256 -extensions v3_req -extfile ssl.cnf

拷贝服务器证书到emqx的服务端

cp ca.pem server-cert.pem server-key.pem /etc/emqx/certs
chown -R emqx.emqx /etc/emqx/certs

5、生成客户端证书

openssl genrsa -out client-key.pem 2048
openssl req -new -key client-key.pem -out client-csr.pem -subj "/C=CN/ST=Zhejiang/L=Hangzhou/O=EMQX/CN=client"
openssl x509 -req -days 3650 -in client-csr.pem -CA ca.pem -CAkey ca.key -CAcreateserial -out client-cert.pem

5、下载并配置emqx

下载并解压

wget https://www.emqx.com/zh/downloads/broker/5.7.1/emqx-5.7.1-el8-amd64.tar.gz
tar zxvf emqx-5.7.1-el8-amd64.tar.gz

# 创建链接
ln -s /opt/emqx571/bin/emqx /usr/bin/emqx

配置环境变量

#vim /etc/profile

PATH=$PATH:/opt/emqx571/bin
export PATH

6、配置emqx.conf

配置过程中，请注意node.name的配置

node {
  name = "emqx@emqx1.aolingo.com"
  cookie = "emqxsecretcookie"
  data_dir = "/var/lib/emqx"
}

cluster {
  name = emqxcl
  discovery_strategy = manual
}

log {
  file {
    enable = true
    formatter = text
    level = info #warning
    path = "/var/log/emqx/emqx.log"
    rotation_count = 10
    rotation_size = 50MB
    time_offset = system
    timestamp_format = auto

  }
}

dashboard {
  listeners.http {
    bind = 18083
  }
}

listeners.ssl.default {
  bind = "0.0.0.0:20442"
  enable_authn = false
  ssl_options {
    cacertfile = "/opt/emqx571/etc/certs/ca.pem"
    certfile = "/opt/emqx571/etc/certs/server-cert.pem"
    keyfile = "/opt/emqx571/etc/certs/server-key.pem"
    verify = verify_peer
    fail_if_no_peer_cert = true
  }
}

listeners.wss.default {
  bind = "0.0.0.0:8084"
  max_connections = 1024000
  websocket.mqtt_path = "/mqtt"
  ssl_options {
    cacertfile = "/opt/emqx571/etc/certs/ca.pem"
    certfile = "/opt/emqx571/etc/certs/server-cert.pem"
    keyfile = "/opt/emqx571/etc/certs/server-key.pem"
  }
}

7、启动每一个节点，并配置集群

在不同的节点执行如下指令。不要互相添加节点，找一个节点作为中心节点，其他节点都执行同一条指令才能保证所有节点添加到一个集群中。

每个节点启动

emqx start

添加到集群

./bin/emqx ctl cluster join emqx@emqx1.aolingo.com

8、配置服务

# vim /lib/systemd/system/emqx.service 

[Unit]
Description=emqx daemon
After=network.target

[Service]
User=root
Group=root

# The ExecStart= is foreground, so 'simple' here
Type=simple
Environment=HOME=/var/lib/emqx

# log to file by default (if no log handler config)
Environment=EMQX_DEFAULT_LOG_HANDLER=file

# Start 'foreground' but not 'start' (daemon) mode.
# Because systemd monitor/restarts 'simple' services
ExecStart=/bin/bash /usr/bin/emqx foreground

# Give EMQX enough file descriptors
LimitNOFILE=1048576

# ExecStop is commented out so systemd will send a SIGTERM when 'systemctl stop'.
# SIGTERM is handled by EMQX and it then performs a graceful shutdown
# emqx stop will ping node, always return 0 to make sure next command will be executed
ExecStop=/bin/bash -c '/usr/bin/emqx stop; exit 0'
# If the process is still running, force kill it
ExecStop=/bin/bash -c 'if [ ps -p $MAINPID >/dev/null 2>&1 ]; then kill -15 $MAINPID; fi'

# Wait long enough before force kill for graceful shutdown
TimeoutStopSec=120s

Restart=on-failure

# Do not restart immediately so the peer nodes in the cluster have
# enough time to handle the 'DOWN' events of this node
RestartSec=120s

[Install]
WantedBy=multi-user.target
systemctl enable emqx
systemctl start emqx

9、一些操作

（1）、更改管理员密码

./bin/emqx ctl admins passwd admin 密码明文