K8S部署Nacos集群 - 部署笔记
1）Nacos集群部署的安装包准备
官方下载的nacos-server-1.2.1.zip包需要经过处理下：修改配置、加入docker-startup.sh启动脚本
[root@k8s-vm01 nacos-cluster]# pwd
/usr/local/src/nacos-cluster
[root@k8s-vm01 nacos-cluster]# ls
nacos-server-1.2.1.zip
[root@k8s-vm01 nacos-cluster]# unzip nacos-server-1.2.1.zip
nacos-server-1.2.1 nacos-server-1.2.1.zip
[root@k8s-vm01 conf]# pwd
/usr/local/src/nacos-cluster/nacos-server-1.2.1/nacos/conf
这里需要对application.properties进行修改：
[root@k8s-vm01 conf]# cat application.properties
# spring
server.servlet.contextPath=${SERVER_SERVLET_CONTEXTPATH:/nacos}
server.contextPath=/nacos
server.port=${NACOS_SERVER_PORT:8848}
spring.datasource.platform=${SPRING_DATASOURCE_PLATFORM:""}
nacos.cmdb.dumpTaskInterval=3600
nacos.cmdb.eventTaskInterval=10
nacos.cmdb.labelTaskInterval=300
nacos.cmdb.loadDataAtStart=false
db.num=${MYSQL_DATABASE_NUM:1}
db.url.0=jdbc:mysql://${MYSQL_SERVICE_HOST}:${MYSQL_SERVICE_PORT:3306}/${MYSQL_SERVICE_DB_NAME}?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true
db.user=${MYSQL_SERVICE_USER}
db.password=${MYSQL_SERVICE_PASSWORD}
### The auth system to use, currently only 'nacos' is supported:
nacos.core.auth.system.type=${NACOS_AUTH_SYSTEM_TYPE:nacos}
### The token expiration in seconds:
nacos.core.auth.default.token.expire.seconds=${NACOS_AUTH_TOKEN_EXPIRE_SECONDS:18000}
### The default token:
nacos.core.auth.default.token.secret.key=${NACOS_AUTH_TOKEN:SecretKey012345678901234567890123456789012345678901234567890123456789}
### Turn on/off caching of auth information. By turning on this switch, the update of auth information would have a 15 seconds delay.
nacos.core.auth.caching.enabled=${NACOS_AUTH_CACHE_ENABLE:false}
server.tomcat.accesslog.enabled=${TOMCAT_ACCESSLOG_ENABLED:false}
server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b %D
# default current work dir
server.tomcat.basedir=
## spring security config
### turn off security
nacos.security.ignore.urls=/,/error,/**/*.css,/**/*.js,/**/*.html,/**/*.map,/**/*.svg,/**/*.png,/**/*.ico,/console-fe/public/**,/v1/auth/**,/v1/console/health/**,/actuator/**,/v1/console/server/**
# metrics for elastic search
management.metrics.export.elastic.enabled=false
management.metrics.export.influx.enabled=false
nacos.naming.distro.taskDispatchThreadCount=10
nacos.naming.distro.taskDispatchPeriod=200
nacos.naming.distro.batchSyncKeyCount=1000
nacos.naming.distro.initDataRatio=0.9
nacos.naming.distro.syncRetryDelay=5000
nacos.naming.data.warmup=true
还需要在bin目录下添加docker-startup.sh启动脚本
容器里nacos集群模式的启动脚本必须使用docker-startup.sh这个，不能使用startup.sh启动脚本
[root@k8s-vm01 bin]# cat docker-startup.sh
#!/bin/bash
# Copyright 1999-2018 Alibaba Group Holding Ltd.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -x
export DEFAULT_SEARCH_LOCATIONS="classpath:/,classpath:/config/,file:./,file:./config/"
export CUSTOM_SEARCH_LOCATIONS=${DEFAULT_SEARCH_LOCATIONS},file:${BASE_DIR}/conf/,${BASE_DIR}/init.d/
export CUSTOM_SEARCH_NAMES="application,custom"
PLUGINS_DIR="/home/nacos/plugins/peer-finder"
function print_servers(){
if [[ ! -d "${PLUGINS_DIR}" ]]; then
echo "" > "$CLUSTER_CONF"
for server in ${NACOS_SERVERS}; do
echo "$server" >> "$CLUSTER_CONF"
done
else
bash $PLUGINS_DIR/plugin.sh
sleep 30
fi
}
#===========================================================================================
# JVM Configuration
#===========================================================================================
if [[ "${MODE}" == "standalone" ]]; then
JAVA_OPT="${JAVA_OPT} -Xms512m -Xmx512m -Xmn256m"
JAVA_OPT="${JAVA_OPT} -Dnacos.standalone=true"
else
JAVA_OPT="${JAVA_OPT} -server -Xms${JVM_XMS} -Xmx${JVM_XMX} -Xmn${JVM_XMN} -XX:MetaspaceSize=${JVM_MS} -XX:MaxMetaspaceSize=${JVM_MMS}"
if [[ "${NACOS_DEBUG}" == "y" ]]; then
JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n"
fi
JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${BASE_DIR}/logs/java_heapdump.hprof"
JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages"
print_servers
fi
#===========================================================================================
# Setting system properties
#===========================================================================================
# set mode that Nacos Server function of split
if [[ "${FUNCTION_MODE}" == "config" ]]; then
JAVA_OPT="${JAVA_OPT} -Dnacos.functionMode=config"
elif [[ "${FUNCTION_MODE}" == "naming" ]]; then
JAVA_OPT="${JAVA_OPT} -Dnacos.functionMode=naming"
fi
# set nacos server ip
if [[ ! -z "${NACOS_SERVER_IP}" ]]; then
JAVA_OPT="${JAVA_OPT} -Dnacos.server.ip=${NACOS_SERVER_IP}"
fi
if [[ ! -z "${USE_ONLY_SITE_INTERFACES}" ]]; then
JAVA_OPT="${JAVA_OPT} -Dnacos.inetutils.use-only-site-local-interfaces=${USE_ONLY_SITE_INTERFACES}"
fi
if [[ ! -z "${PREFERRED_NETWORKS}" ]]; then
JAVA_OPT="${JAVA_OPT} -Dnacos.inetutils.preferred-networks=${PREFERRED_NETWORKS}"
fi
if [[ ! -z "${IGNORED_INTERFACES}" ]]; then
JAVA_OPT="${JAVA_OPT} -Dnacos.inetutils.ignored-interfaces=${IGNORED_INTERFACES}"
fi
### If turn on auth system:
if [[ ! -z "${NACOS_AUTH_ENABLE}" ]]; then
JAVA_OPT="${JAVA_OPT} -Dnacos.core.auth.enabled=${NACOS_AUTH_ENABLE}"
fi
if [[ "${PREFER_HOST_MODE}" == "hostname" ]]; then
JAVA_OPT="${JAVA_OPT} -Dnacos.preferHostnameOverIp=true"
fi
JAVA_MAJOR_VERSION=$($JAVA -version 2>&1 | sed -E -n 's/.* version "([0-9]*).*$/\1/p')
if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then
JAVA_OPT="${JAVA_OPT} -cp .:${BASE_DIR}/plugins/cmdb/*.jar:${BASE_DIR}/plugins/mysql/*.jar"
JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${BASE_DIR}/logs/nacos_gc.log:time,tags:filecount=10,filesize=102400"
else
JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${JAVA_HOME}/lib/ext:${BASE_DIR}/plugins/health:${BASE_DIR}/plugins/cmdb:${BASE_DIR}/plugins/mysql"
JAVA_OPT="${JAVA_OPT} -Xloggc:${BASE_DIR}/logs/nacos_gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M"
fi
JAVA_OPT="${JAVA_OPT} -Dnacos.home=${BASE_DIR}"
JAVA_OPT="${JAVA_OPT} -jar ${BASE_DIR}/target/nacos-server.jar"
JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}"
JAVA_OPT="${JAVA_OPT} --spring.config.location=${CUSTOM_SEARCH_LOCATIONS}"
JAVA_OPT="${JAVA_OPT} --spring.config.name=${CUSTOM_SEARCH_NAMES}"
JAVA_OPT="${JAVA_OPT} --logging.config=${BASE_DIR}/conf/nacos-logback.xml"
JAVA_OPT="${JAVA_OPT} --server.max-http-header-size=524288"
echo "nacos is starting,you can check the ${BASE_DIR}/logs/start.out"
echo "$JAVA ${JAVA_OPT}" > ${BASE_DIR}/logs/start.out 2>&1 &
nohup $JAVA ${JAVA_OPT} > ${BASE_DIR}/logs/start.out 2>&1 < /dev/null
修改后，再将nacos-server-1.2.1目录打包成nacos-server-1.2.1.tar.gz
[root@k8s-vm01 nacos-cluster]# tar -zvcf nacos-server-1.2.1.tar.gz nacos-server-1.2.1
[root@k8s-vm01 nacos-cluster]# ls
nacos-server-1.2.1 nacos-server-1.2.1.tar.gz nacos-server-1.2.1.zip
2）Nacos镜像制作
[root@k8s-vm01 nacos-cluster]# cat Dockerfile FROM 192.168.1.75/wise-ops/jdk1.8.0_192:latest RUN rm -f /etc/localtime \ && ln -sv /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ && echo "Asia/Shanghai" > /etc/timezone ENV LANG en_US.UTF-8 ENV MODE cluster ENV PREFER_HOST_MODE ip ENV BASE_DIR /home/nacos ENV CLASSPATH .:/home/nacos/conf: ENV CLUSTER_CONF /home/nacos/conf/cluster.conf ENV FUNCTION_MODE all ENV JAVA_HOME /usr/java/jdk1.8.0_192 ENV NACOS_USER nacos ENV JAVA /usr/java/jdk1.8.0_192/bin/java ENV JVM_XMS 2g ENV JVM_XMX 2g ENV JVM_XMN 1g ENV JVM_MS 128m ENV JVM_MMS 320m ENV NACOS_DEBUG n ENV TOMCAT_ACCESSLOG_ENABLED false WORKDIR /home/nacos ADD nacos-server-1.2.1.tar.gz /home RUN set -x && mv /home/nacos-server-1.2.1/nacos/* /home/nacos/ && rm -rf /home/nacos-server-1.2.1 RUN mkdir -p logs && cd logs && touch start.out && ln -sf /dev/stdout start.out && ln -sf /dev/stderr start.out RUN chmod 755 bin/docker-startup.sh EXPOSE 8848 ENTRYPOINT ["bin/docker-startup.sh"]
制作镜像并上传Harbor
[root@k8s-vm01 nacos-cluster]# docker build -t 192.168.1.75/wise-ops/nacos-cluster:v1.0 . [root@k8s-vm01 nacos-cluster]# docker push 192.168.1.75/wise-ops/nacos-cluster:v1.0
3）部署Nacos集群
这里采用了configmap存储卷，将mysql配置信息存到了configmap中
注意：需要提前在mysql数据库中创建一个nacos库名！然后将上面nacos-server-1.2.1.tar.gz包中的conf/nacos-mysql.sql文件里的sql语句在mysql的nacos库下执行（source nacos-mysql.sql ）导入语句。
[root@k8s-vm01 nacos-cluster]# pwd
/opt/k8s/work/test_yml/nacos-cluster
[root@k8s-vm01 nacos-cluster]# cat nacos-cluster.yml
---
apiVersion: v1
kind: Service
metadata:
namespace: wise
name: nacos-cluster
labels:
app: nacos-cluster
spec:
ports:
- port: 8848
name: server
targetPort: 8848
clusterIP: None
selector:
app: nacos-cluster
---
apiVersion: v1
kind: ConfigMap
metadata:
namespace: wise
name: nacos-cluster-cm
data:
mysql.host: "192.168.1.72"
mysql.db.name: "nacos"
mysql.port: "3306"
mysql.user: "nacos"
mysql.password: "nacos@123"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
namespace: wise
name: nacos-cluster
spec:
serviceName: nacos-cluster
replicas: 3
template:
metadata:
labels:
app: nacos-cluster
annotations:
pod.alpha.kubernetes.io/initialized: "true"
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- nacos-cluster
topologyKey: "kubernetes.io/hostname"
containers:
- name: k8snacos
imagePullPolicy: Always
image: 192.168.1.75/wise-ops/nacos-cluster:v10
resources:
requests:
memory: 2048Mi
cpu: 1000m
limits:
memory: 2048Mi
cpu: 1000m
ports:
- containerPort: 8848
name: client
env:
- name: NACOS_REPLICAS
value: "3"
- name: MYSQL_SERVICE_HOST
valueFrom:
configMapKeyRef:
name: nacos-cluster-cm
key: mysql.host
- name: MYSQL_SERVICE_DB_NAME
valueFrom:
configMapKeyRef:
name: nacos-cluster-cm
key: mysql.db.name
- name: MYSQL_SERVICE_PORT
valueFrom:
configMapKeyRef:
name: nacos-cluster-cm
key: mysql.port
- name: MYSQL_SERVICE_USER
valueFrom:
configMapKeyRef:
name: nacos-cluster-cm
key: mysql.user
- name: MYSQL_SERVICE_PASSWORD
valueFrom:
configMapKeyRef:
name: nacos-cluster-cm
key: mysql.password
- name: NACOS_SERVER_PORT
value: "8848"
- name: PREFER_HOST_MODE
value: "hostname"
- name: NACOS_SERVERS
value: "nacos-cluster-0.nacos-cluster.wise.svc.cluster.local:8848 nacos-cluster-1.nacos-cluster.wise.svc.cluster.local:8848 nacos-cluster-2.nacos-cluster.wise.svc.cluster.local:8848"
selector:
matchLabels:
app: nacos-cluster
创建并查看
[root@k8s-vm01 nacos-cluster]# kubectl create -f nacos-cluster.yml service/nacos-cluster created configmap/nacos-cluster-cm created statefulset.apps/nacos-cluster created [root@k8s-vm01 ~]# kubectl get pods -n wise|grep nacos-cluster nacos-cluster-0 1/1 Running 0 49m nacos-cluster-1 1/1 Running 0 49m nacos-cluster-2 1/1 Running 0 48m [root@k8s-vm01 ~]# kubectl get svc -n wise|grep nacos-cluster nacos-cluster ClusterIP None <none> 8848/TCP 50m [root@k8s-vm01 ~]# kubectl get statefulset -n wise NAME READY AGE nacos-cluster 3/3 50m [root@k8s-vm01 ~]# kubectl get configmap -n wise|grep nacos-cluster nacos-cluster-cm 5 50m
配置ingress，外部访问nacos
目录 返回
首页