何宇泽

k8s部署nacos集群

发表于 更新于 分类于
本文字数： 13k 阅读时长 ≈ 12 分钟

1. k8s部署nacos集群

1.1. Nacos集群部署的安装包准备

下载nacos安装包

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[root@k8s-master data]# pwd
/data/nacos-server-2.0.2

# 下载安装包
[root@k8s-master data]# wget https://github.com/alibaba/nacos/releases/download/2.0.2/nacos-server-2.0.2.zip

# 解压
[root@k8s-master data]# unzip nacos-server-2.0.2.zip
[root@k8s-master data]# mv nacos nacos-server-2.0.2

# 进入配置文件目录
[root@k8s-master conf]# cd nacos-server-2.0.2/nacos/conf

# 备份配置文件
[root@k8s-master conf]# mv application.properties application.properties.bak

修改配置文件

[root@k8s-master conf]# cat application.properties

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
# spring
server.servlet.contextPath=${SERVER_SERVLET_CONTEXTPATH:/nacos}
server.contextPath=/nacos
server.port=${NACOS_SERVER_PORT:8848}
spring.datasource.platform=${SPRING_DATASOURCE_PLATFORM:""}
nacos.cmdb.dumpTaskInterval=3600
nacos.cmdb.eventTaskInterval=10
nacos.cmdb.labelTaskInterval=300
nacos.cmdb.loadDataAtStart=false
db.num=${MYSQL_DATABASE_NUM:1}
db.url.0=jdbc:mysql://${MYSQL_SERVICE_HOST}:${MYSQL_SERVICE_PORT:3306}/${MYSQL_SERVICE_DB_NAME}?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true
db.user=${MYSQL_SERVICE_USER}
db.password=${MYSQL_SERVICE_PASSWORD}
### The auth system to use, currently only 'nacos' is supported:
nacos.core.auth.system.type=${NACOS_AUTH_SYSTEM_TYPE:nacos}
 
 
### The token expiration in seconds:
nacos.core.auth.default.token.expire.seconds=${NACOS_AUTH_TOKEN_EXPIRE_SECONDS:18000}
 
### The default token:
nacos.core.auth.default.token.secret.key=${NACOS_AUTH_TOKEN:SecretKey012345678901234567890123456789012345678901234567890123456789}
 
### Turn on/off caching of auth information. By turning on this switch, the update of auth information would have a 15 seconds delay.
nacos.core.auth.caching.enabled=${NACOS_AUTH_CACHE_ENABLE:false}
 
server.tomcat.accesslog.enabled=${TOMCAT_ACCESSLOG_ENABLED:false}
server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b %D
# default current work dir
server.tomcat.basedir=
## spring security config
### turn off security
nacos.security.ignore.urls=/,/error,/**/*.css,/**/*.js,/**/*.html,/**/*.map,/**/*.svg,/**/*.png,/**/*.ico,/console-fe/public/**,/v1/auth/**,/v1/console/health/**,/actuator/**,/v1/console/server/**
# metrics for elastic search
management.metrics.export.elastic.enabled=false
management.metrics.export.influx.enabled=false
 
nacos.naming.distro.taskDispatchThreadCount=10
nacos.naming.distro.taskDispatchPeriod=200
nacos.naming.distro.batchSyncKeyCount=1000
nacos.naming.distro.initDataRatio=0.9
nacos.naming.distro.syncRetryDelay=5000
nacos.naming.data.warmup=true

添加docker-startup.sh脚本

容器里nacos集群模式的启动脚本必须使用docker-startup.sh，不能使用startup.sh启动脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
[root@k8s-master conf]# cd ../bin/


[root@k8s-master bin]# cat docker-startup.sh

#!/bin/bash
# Copyright 1999-2018 Alibaba Group Holding Ltd.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
 
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -x
export DEFAULT_SEARCH_LOCATIONS="classpath:/,classpath:/config/,file:./,file:./config/"
export CUSTOM_SEARCH_LOCATIONS=${DEFAULT_SEARCH_LOCATIONS},file:${BASE_DIR}/conf/,${BASE_DIR}/init.d/
export CUSTOM_SEARCH_NAMES="application,custom"
PLUGINS_DIR="/home/nacos/plugins/peer-finder"
function print_servers(){
   if [[ ! -d "${PLUGINS_DIR}" ]]; then
    echo "" > "$CLUSTER_CONF"
    for server in ${NACOS_SERVERS}; do
            echo "$server" >> "$CLUSTER_CONF"
    done
   else
    bash $PLUGINS_DIR/plugin.sh
   sleep 30
        fi
}
#===========================================================================================
# JVM Configuration
#===========================================================================================
if [[ "${MODE}" == "standalone" ]]; then
 
    JAVA_OPT="${JAVA_OPT} -Xms512m -Xmx512m -Xmn256m"
    JAVA_OPT="${JAVA_OPT} -Dnacos.standalone=true"
else
 
  JAVA_OPT="${JAVA_OPT} -server -Xms${JVM_XMS} -Xmx${JVM_XMX} -Xmn${JVM_XMN} -XX:MetaspaceSize=${JVM_MS} -XX:MaxMetaspaceSize=${JVM_MMS}"
  if [[ "${NACOS_DEBUG}" == "y" ]]; then
    JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n"
  fi
  JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${BASE_DIR}/logs/java_heapdump.hprof"
  JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages"
  print_servers
fi
 
#===========================================================================================
# Setting system properties
#===========================================================================================
# set  mode that Nacos Server function of split
if [[ "${FUNCTION_MODE}" == "config" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.functionMode=config"
elif [[ "${FUNCTION_MODE}" == "naming" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.functionMode=naming"
fi
# set nacos server ip
if [[ ! -z "${NACOS_SERVER_IP}" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.server.ip=${NACOS_SERVER_IP}"
fi
 
if [[ ! -z "${USE_ONLY_SITE_INTERFACES}" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.inetutils.use-only-site-local-interfaces=${USE_ONLY_SITE_INTERFACES}"
fi
 
if [[ ! -z "${PREFERRED_NETWORKS}" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.inetutils.preferred-networks=${PREFERRED_NETWORKS}"
fi
 
if [[ ! -z "${IGNORED_INTERFACES}" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.inetutils.ignored-interfaces=${IGNORED_INTERFACES}"
fi
 
### If turn on auth system:
if [[ ! -z "${NACOS_AUTH_ENABLE}" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.core.auth.enabled=${NACOS_AUTH_ENABLE}"
fi
 
if [[ "${PREFER_HOST_MODE}" == "hostname" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.preferHostnameOverIp=true"
fi
 
JAVA_MAJOR_VERSION=$($JAVA -version 2>&1 | sed -E -n 's/.* version "([0-9]*).*$/\1/p')
if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then
  JAVA_OPT="${JAVA_OPT} -cp .:${BASE_DIR}/plugins/cmdb/*.jar:${BASE_DIR}/plugins/mysql/*.jar"
  JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${BASE_DIR}/logs/nacos_gc.log:time,tags:filecount=10,filesize=102400"
else
  JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${JAVA_HOME}/lib/ext:${BASE_DIR}/plugins/health:${BASE_DIR}/plugins/cmdb:${BASE_DIR}/plugins/mysql"
  JAVA_OPT="${JAVA_OPT} -Xloggc:${BASE_DIR}/logs/nacos_gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M"
fi
 
 
 
JAVA_OPT="${JAVA_OPT} -Dnacos.home=${BASE_DIR}"
JAVA_OPT="${JAVA_OPT} -jar ${BASE_DIR}/target/nacos-server.jar"
JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}"
JAVA_OPT="${JAVA_OPT} --spring.config.location=${CUSTOM_SEARCH_LOCATIONS}"
JAVA_OPT="${JAVA_OPT} --spring.config.name=${CUSTOM_SEARCH_NAMES}"
JAVA_OPT="${JAVA_OPT} --logging.config=${BASE_DIR}/conf/nacos-logback.xml"
JAVA_OPT="${JAVA_OPT} --server.max-http-header-size=524288"
 
echo "nacos is starting,you can check the ${BASE_DIR}/logs/start.out"
echo "$JAVA ${JAVA_OPT}" > ${BASE_DIR}/logs/start.out 2>&1 &
nohup $JAVA ${JAVA_OPT} > ${BASE_DIR}/logs/start.out 2>&1 < /dev/null

打包

1
2
3
4
[root@k8s-master bin]# cd /data
[root@k8s-master data]# tar -zvcf nacos-server-2.0.2.tar.gz nacos-server-2.0.2
[root@k8s-master data]# ls
nacos-server-2.0.2  nacos-server-2.0.2.tar.gz  nacos-server-2.0.2.zip

1.2. Nacos镜像制作

编写dockerfile

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
[root@k8s-master data]# cat Dockerfile

FROM heyuze/jdk1.8.0_301:latest
RUN rm -f /etc/localtime \
&& ln -sv /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone
 
ENV LANG en_US.UTF-8
 
ENV MODE cluster
ENV PREFER_HOST_MODE ip
ENV BASE_DIR /home/nacos
ENV CLASSPATH .:/home/nacos/conf:
ENV CLUSTER_CONF /home/nacos/conf/cluster.conf
ENV FUNCTION_MODE all
ENV JAVA_HOME /usr/local/jdk/jdk1.8.0_301
ENV NACOS_USER nacos
ENV JAVA /usr/local/jdk/jdk1.8.0_301/bin/java
ENV JVM_XMS 2g
ENV JVM_XMX 2g
ENV JVM_XMN 1g
ENV JVM_MS 128m
ENV JVM_MMS 320m
ENV NACOS_DEBUG n
ENV TOMCAT_ACCESSLOG_ENABLED false
 
WORKDIR /home/nacos
ADD nacos-server-2.0.2.tar.gz /home
RUN set -x && mv /home/nacos-server-2.0.2/nacos/* /home/nacos/ && rm -rf /home/nacos-server-2.0.2
 
RUN mkdir -p logs && cd logs && touch start.out && ln -sf /dev/stdout start.out && ln -sf /dev/stderr start.out
RUN chmod 755 bin/docker-startup.sh
 
EXPOSE 8848
ENTRYPOINT ["bin/docker-startup.sh"]

制作镜像并上传

1
2
[root@k8s-master data]# docker build -t heyuze/nacos-cluster:v2.0.2 .
[root@k8s-master data]# docker push heyuze/nacos-cluster:v2.0.2

1.3. 部署Nacos集群

这里采用了configmap存储卷，将mysql配置信息存到了configmap中

注意：需要提前在mysql数据库中创建一个nacos库名！然后将上面nacos-server-2.0.2.tar.gz包中的conf/nacos-mysql.sql文件里的sql语句在mysql的nacos库下执行（source nacos-mysql.sql ）导入语句。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
[root@k8s-master nacos_cluster]# pwd
/data/k8s-yaml/test_yaml/nacos_cluster

[root@k8s-master nacos_cluster]# cat nacos-cluster.yaml

---
apiVersion: v1
kind: Service
metadata:
  namespace: wise
  name: nacos-cluster
  labels:
    app: nacos-cluster
spec:
  ports:
    - port: 8848
      name: server
      targetPort: 8848
  clusterIP: None
  selector:
    app: nacos-cluster
---
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: wise
  name: nacos-cluster-cm
data:
  mysql.host: "192.168.101.221"
  mysql.db.name: "nacos"
  mysql.port: "3306"
  mysql.user: "nacos"
  mysql.password: "nacos@123"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  namespace: wise
  name: nacos-cluster
spec:
  serviceName: nacos-cluster
  replicas: 3
  template:
    metadata:
      labels:
        app: nacos-cluster
      annotations:
        pod.alpha.kubernetes.io/initialized: "true"
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - nacos-cluster
              topologyKey: "kubernetes.io/hostname"
      containers:
        - name: k8snacos
          imagePullPolicy: Always
          image: heyuze/nacos-cluster:v2.0.2
          resources:
            requests:
              memory: 2048Mi
              cpu: 1000m
            limits:
              memory: 2048Mi
              cpu: 1000m
          ports:
            - containerPort: 8848
              name: client
          env:
            - name: NACOS_REPLICAS
              value: "3"
            - name: MYSQL_SERVICE_HOST
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.host
            - name: MYSQL_SERVICE_DB_NAME
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.db.name
            - name: MYSQL_SERVICE_PORT
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.port
            - name: MYSQL_SERVICE_USER
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.user
            - name: MYSQL_SERVICE_PASSWORD
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.password
            - name: NACOS_SERVER_PORT
              value: "8848"
            - name: PREFER_HOST_MODE
              value: "hostname"
            - name: NACOS_SERVERS
              value: "nacos-cluster-0.nacos-cluster.wise.svc.cluster.local:8848 nacos-cluster-1.nacos-cluster.wise.svc.cluster.local:8848 nacos-cluster-2.nacos-cluster.wise.svc.cluster.local:8848"
  selector:
    matchLabels:
      app: nacos-cluster

创建并查看

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
[root@k8s-master nacos_cluster]# kubectl create -f nacos-cluster.yaml 
service/nacos-cluster created
configmap/nacos-cluster-cm created
statefulset.apps/nacos-cluster created

[root@k8s-master nacos_cluster]# kubectl get pods -n wise|grep nacos-cluster
nacos-cluster-0   1/1     Running   0          12m
nacos-cluster-1   1/1     Running   0          11m
nacos-cluster-2   1/1     Running   0          10m

[root@k8s-master nacos_cluster]# kubectl get svc -n wise|grep nacos-cluster
nacos-cluster   ClusterIP   None         <none>        8848/TCP   12m

[root@k8s-master nacos_cluster]# kubectl get statefulset -n wise
NAME            READY   AGE
nacos-cluster   3/3     13m

[root@k8s-master nacos_cluster]# kubectl get configmap -n wise|grep nacos-cluster
nacos-cluster-cm   5      13m

创建ingress访问

vim nacos-ingress.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  annotations:
  name: nacos-cluster
  namespace: wise
  annotations:
    kubernetes.io/ingress.class: traefik-v2.3     
spec:
  rules:
  - host: nacos.heyonggs.com
    http:
      paths:
      - backend:
          serviceName: nacos-cluster
          servicePort: 8848

应用

1
kubectl apply -f nacos-ingress.yaml

先做域名解析在访问查看集群：http://域名/nacos

账号：nacos

密码：nacos

traefik

-------------本文结束感谢您的阅读-------------