· 6 years ago · Oct 20, 2019, 03:32 PM
1INFORMATION ON VM:
2
3[root@kubemaster ~]# cat /etc/redhat-release
4CentOS Linux release 7.7.1908 (Core)
5[root@kubemaster ~]# docker version
6Client: Docker Engine - Community
7 Version: 19.03.4
8 API version: 1.40
9 Go version: go1.12.10
10 Git commit: 9013bf583a
11 Built: Fri Oct 18 15:52:22 2019
12 OS/Arch: linux/amd64
13 Experimental: false
14
15Server: Docker Engine - Community
16 Engine:
17 Version: 19.03.4
18 API version: 1.40 (minimum version 1.12)
19 Go version: go1.12.10
20 Git commit: 9013bf583a
21 Built: Fri Oct 18 15:50:54 2019
22 OS/Arch: linux/amd64
23 Experimental: false
24 containerd:
25 Version: 1.2.10
26 GitCommit: b34a5c8af56e510852c35414db4c1f4fa6172339
27 runc:
28 Version: 1.0.0-rc8+dev
29 GitCommit: 3e425f80a8c931f88e6d94a8c831b9d5aa481657
30 docker-init:
31 Version: 0.18.0
32 GitCommit: fec3683
33[root@kubemaster ~]# kube
34kubeadm kubectl kubelet
35[root@kubemaster ~]# yum list installed | grep -i kube
36Failed to set locale, defaulting to C
37cri-tools.x86_64 1.13.0-0 @kubernetes
38kubeadm.x86_64 1.16.2-0 @kubernetes
39kubectl.x86_64 1.16.2-0 @kubernetes
40kubelet.x86_64 1.16.2-0 @kubernetes
41kubernetes-cni.x86_64 0.7.5-0 @kubernetes
42
43[root@kubemaster ~]# ip a s
441: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
45 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
46 inet 127.0.0.1/8 scope host lo
47 valid_lft forever preferred_lft forever
48 inet6 ::1/128 scope host
49 valid_lft forever preferred_lft forever
502: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
51 link/ether 08:00:27:f6:23:fb brd ff:ff:ff:ff:ff:ff
52 inet 192.168.2.46/24 brd 192.168.2.255 scope global noprefixroute enp0s3
53 valid_lft forever preferred_lft forever
54 inet6 fe80::98de:d615:c004:9c0a/64 scope link tentative noprefixroute dadfailed
55 valid_lft forever preferred_lft forever
56 inet6 fe80::ccba:4f5a:d199:7736/64 scope link noprefixroute
57 valid_lft forever preferred_lft forever
583: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
59 link/ether 02:42:07:56:fd:a2 brd ff:ff:ff:ff:ff:ff
60 inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
61 valid_lft forever preferred_lft forever
62
63-----------------------------------------------------------------------------------
64
65[root@kubemaster ~]# kubeadm init --apiserver-advertise-address=192.168.2.46 --pod-network-cidr=192.168.0.0/16
66[init] Using Kubernetes version: v1.16.2
67[preflight] Running pre-flight checks
68 [WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
69 [WARNING SystemVerification]: this Docker version is not on the list of validated versions: 19.03.4. Latest validated version: 18.09
70[preflight] Pulling images required for setting up a Kubernetes cluster
71[preflight] This might take a minute or two, depending on the speed of your internet connection
72[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
73[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
74[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
75[kubelet-start] Activating the kubelet service
76[certs] Using certificateDir folder "/etc/kubernetes/pki"
77[certs] Generating "ca" certificate and key
78[certs] Generating "apiserver" certificate and key
79[certs] apiserver serving cert is signed for DNS names [kubemaster kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.2.46]
80[certs] Generating "apiserver-kubelet-client" certificate and key
81[certs] Generating "front-proxy-ca" certificate and key
82[certs] Generating "front-proxy-client" certificate and key
83[certs] Generating "etcd/ca" certificate and key
84[certs] Generating "etcd/server" certificate and key
85[certs] etcd/server serving cert is signed for DNS names [kubemaster localhost] and IPs [192.168.2.46 127.0.0.1 ::1]
86[certs] Generating "etcd/peer" certificate and key
87[certs] etcd/peer serving cert is signed for DNS names [kubemaster localhost] and IPs [192.168.2.46 127.0.0.1 ::1]
88[certs] Generating "etcd/healthcheck-client" certificate and key
89[certs] Generating "apiserver-etcd-client" certificate and key
90[certs] Generating "sa" key and public key
91[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
92[kubeconfig] Writing "admin.conf" kubeconfig file
93[kubeconfig] Writing "kubelet.conf" kubeconfig file
94[kubeconfig] Writing "controller-manager.conf" kubeconfig file
95[kubeconfig] Writing "scheduler.conf" kubeconfig file
96[control-plane] Using manifest folder "/etc/kubernetes/manifests"
97[control-plane] Creating static Pod manifest for "kube-apiserver"
98[control-plane] Creating static Pod manifest for "kube-controller-manager"
99[control-plane] Creating static Pod manifest for "kube-scheduler"
100[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
101[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
102[kubelet-check] Initial timeout of 40s passed.
103^C
104
105==============================================================================
106
107/var/log/messages
108
109Oct 20 10:11:25 kubemaster systemd: Started Kubernetes systemd probe.
110Oct 20 10:11:26 kubemaster kubelet: I1020 10:11:25.975063 30409 server.go:410] Version: v1.16.2
111Oct 20 10:11:26 kubemaster kubelet: I1020 10:11:25.976501 30409 plugins.go:100] No cloud provider specified.
112Oct 20 10:11:26 kubemaster kubelet: W1020 10:11:25.976682 30409 server.go:549] standalone mode, no API client
113Oct 20 10:11:27 kubemaster kubelet: W1020 10:11:27.544940 30409 server.go:467] No api server defined - no events will be sent to API server.
114Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.545040 30409 server.go:636] --cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /
115Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.555561 30409 container_manager_linux.go:265] container manager verified user specified cgroup-root exists: []
116Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.555654 30409 container_manager_linux.go:270] Creating Container Manager object based on Node Config: {RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: ContainerRuntime:docker CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:false NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.1} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.inodesFree Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.15} GracePeriod:0s MinReclaim:<nil>} {Signal:memory.available Operator:LessThan Value:{Quantity:100Mi Percentage:0} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalCPUManagerReconcilePeriod:10s ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none}
117Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.556459 30409 fake_topology_manager.go:29] [fake topologymanager] NewFakeManager
118Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.556497 30409 container_manager_linux.go:305] Creating device plugin manager: true
119Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.557047 30409 fake_topology_manager.go:39] [fake topologymanager] AddHintProvider HintProvider: &{kubelet.sock /var/lib/kubelet/device-plugins/ map[] {0 0} <nil> {{} [0 0 0]} 0x1b6baf0 0x799f318 0x1b6c4f0 map[] map[] map[] map[] map[] 0xc00091ea50 [0] 0x799f318}
120Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.557715 30409 state_mem.go:36] [cpumanager] initializing new in-memory state store
121Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.592560 30409 fake_topology_manager.go:39] [fake topologymanager] AddHintProvider HintProvider: &{{0 0} 0x799f318 10000000000 0xc0005ffc20 <nil> <nil> <nil> <nil> map[memory:{{104857600 0} {<nil>} BinarySI}]}
122Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.655151 30409 client.go:75] Connecting to docker on unix:///var/run/docker.sock
123Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.655386 30409 client.go:104] Start docker client with request timeout=2m0s
124Oct 20 10:11:27 kubemaster kubelet: W1020 10:11:27.803623 30409 docker_service.go:563] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to "hairpin-veth"
125Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.803770 30409 docker_service.go:240] Hairpin mode set to "hairpin-veth"
126Oct 20 10:11:27 kubemaster kubelet: W1020 10:11:27.804737 30409 cni.go:237] Unable to update cni config: no networks found in /etc/cni/net.d
127Oct 20 10:11:27 kubemaster kubelet: I1020 10:11:27.921897 30409 docker_service.go:255] Docker cri networking managed by kubernetes.io/no-op
128Oct 20 10:11:28 kubemaster kubelet: I1020 10:11:28.079410 30409 docker_service.go:260] Docker Info: &{ID:RKYU:GMU7:AA2Q:4RZZ:NQNT:Y7QH:34G7:OQPX:LZR3:22OP:S3BY:QASN Containers:0 ContainersRunning:0 ContainersPaused:0 ContainersStopped:0 Images:12 Driver:overlay2 DriverStatus:[[Backing Filesystem xfs] [Supports d_type true] [Native Overlay Diff true]] SystemStatus:[] Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:[] Log:[awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:true KernelMemoryTCP:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:true IPv4Forwarding:true BridgeNfIptables:true BridgeNfIP6tables:true Debug:false NFd:22 OomKillDisable:true NGoroutines:35 SystemTime:2019-10-20T10:11:27.929428827-04:00 LoggingDriver:json-file CgroupDriver:cgroupfs NEventsListener:0 KernelVersion:3.10.0-1062.1.2.el7.x86_64 OperatingSystem:CentOS Linux 7 (Core) OSType:linux Architecture:x86_64 IndexServerAddress:https://index.docker.io/v1/ RegistryConfig:0xc000731570 NCPU:2 MemTotal:1927163904 GenericResources:[] DockerRootDir:/var/lib/docker HTTPProxy: HTTPSProxy: NoProxy: Name:kubemaster Labels:[] ExperimentalBuild:false ServerVersion:19.03.4 ClusterStore: ClusterAdvertise: Runtimes:map[runc:{Path:runc Args:[]}] DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:[] Nodes:0 Managers:0 Cluster:<nil> Warnings:[]} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:b34a5c8af56e510852c35414db4c1f4fa6172339 Expected:b34a5c8af56e510852c35414db4c1f4fa6172339} RuncCommit:{ID:3e425f80a8c931f88e6d94a8c831b9d5aa481657 Expected:3e425f80a8c931f88e6d94a8c831b9d5aa481657} InitCommit:{ID:fec3683 Expected:fec3683} SecurityOptions:[name=seccomp,profile=default] ProductLicense: Warnings:[]}
129Oct 20 10:11:28 kubemaster kubelet: I1020 10:11:28.079628 30409 docker_service.go:273] Setting cgroupDriver to cgroupfs
130Oct 20 10:11:28 kubemaster kubelet: I1020 10:11:28.428610 30409 remote_runtime.go:59] parsed scheme: ""
131Oct 20 10:11:28 kubemaster kubelet: I1020 10:11:28.428662 30409 remote_runtime.go:59] scheme "" not registered, fallback to default scheme
132Oct 20 10:11:28 kubemaster kubelet: I1020 10:11:28.429022 30409 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock 0 <nil>}] <nil>}
133Oct 20 10:11:28 kubemaster kubelet: I1020 10:11:28.429227 30409 clientconn.go:577] ClientConn switching balancer to "pick_first"
134Oct 20 10:11:28 kubemaster kubelet: I1020 10:11:28.436464 30409 remote_image.go:50] parsed scheme: ""
135Oct 20 10:11:28 kubemaster kubelet: I1020 10:11:28.436540 30409 remote_image.go:50] scheme "" not registered, fallback to default scheme
136Oct 20 10:11:28 kubemaster kubelet: I1020 10:11:28.436605 30409 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock 0 <nil>}] <nil>}
137Oct 20 10:11:28 kubemaster kubelet: I1020 10:11:28.436644 30409 clientconn.go:577] ClientConn switching balancer to "pick_first"
138Oct 20 10:11:48 kubemaster kubelet: E1020 10:11:48.761971 30409 aws_credentials.go:77] while getting AWS credentials NoCredentialProviders: no valid providers in chain. Deprecated.
139Oct 20 10:11:48 kubemaster kubelet: For verbose messaging see aws.Config.CredentialsChainVerboseErrors
140Oct 20 10:11:48 kubemaster kubelet: I1020 10:11:48.904976 30409 kuberuntime_manager.go:207] Container runtime docker initialized, version: 19.03.4, apiVersion: 1.40.0
141Oct 20 10:11:48 kubemaster kubelet: W1020 10:11:48.906480 30409 volume_host.go:79] kubeClient is nil. Skip initialization of CSIDriverLister
142Oct 20 10:11:48 kubemaster kubelet: W1020 10:11:48.915071 30409 csi_plugin.go:198] kubernetes.io/csi: kubeclient not set, assuming standalone kubelet
143Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.053629 30409 server.go:1065] Started kubelet
144Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.056848 30409 server.go:145] Starting to listen on 0.0.0.0:10250
145Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.061987 30409 server.go:354] Adding debug handlers to kubelet server.
146Oct 20 10:11:49 kubemaster kubelet: E1020 10:11:49.071186 30409 kubelet.go:1302] Image garbage collection failed once. Stats initialization may not have completed yet: failed to get imageFs info: unable to find data in memory cache
147Oct 20 10:11:49 kubemaster kubelet: W1020 10:11:49.071884 30409 kubelet.go:1403] No api server defined - no node status update will be sent.
148Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.082310 30409 fs_resource_analyzer.go:64] Starting FS ResourceAnalyzer
149Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.082671 30409 status_manager.go:152] Kubernetes client is nil, not starting status manager.
150Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.082842 30409 kubelet.go:1822] Starting kubelet main sync loop.
151Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.082966 30409 kubelet.go:1839] skipping pod synchronization - [container runtime status check may not have completed yet, PLEG is not healthy: pleg has yet to be successful]
152Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.082977 30409 volume_manager.go:249] Starting Kubelet Volume Manager
153Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.083086 30409 desired_state_of_world_populator.go:131] Desired state populator starts to run
154Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.188686 30409 kubelet.go:1839] skipping pod synchronization - [container runtime status check may not have completed yet, PLEG is not healthy: pleg has yet to be successful]
155Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.202275 30409 reconciler.go:154] Reconciler: start to sync state
156Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.389245 30409 kubelet.go:1839] skipping pod synchronization - container runtime status check may not have completed yet
157Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.467250 30409 clientconn.go:104] parsed scheme: "unix"
158Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.467306 30409 clientconn.go:104] scheme "unix" not registered, fallback to default scheme
159Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.482956 30409 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{unix:///run/containerd/containerd.sock 0 <nil>}] <nil>}
160Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.483024 30409 clientconn.go:577] ClientConn switching balancer to "pick_first"
161Oct 20 10:11:49 kubemaster kubelet: I1020 10:11:49.812552 30409 kubelet.go:1839] skipping pod synchronization - container runtime status check may not have completed yet
162Oct 20 10:11:50 kubemaster kubelet: I1020 10:11:50.317316 30409 kubelet_node_status.go:286] Setting node annotation to enable volume controller attach/detach
163Oct 20 10:11:50 kubemaster kubelet: I1020 10:11:50.616845 30409 kubelet.go:1839] skipping pod synchronization - container runtime status check may not have completed yet
164Oct 20 10:11:50 kubemaster kubelet: I1020 10:11:50.729456 30409 cpu_manager.go:161] [cpumanager] starting with none policy
165Oct 20 10:11:50 kubemaster kubelet: I1020 10:11:50.729508 30409 cpu_manager.go:162] [cpumanager] reconciling every 10s
166Oct 20 10:11:50 kubemaster kubelet: I1020 10:11:50.729633 30409 policy_none.go:42] [cpumanager] none policy: Start
167Oct 20 10:11:50 kubemaster kubelet: W1020 10:11:50.735946 30409 manager.go:569] Failed to retrieve checkpoint for "kubelet_internal_checkpoint": checkpoint is not found
168Oct 20 10:11:50 kubemaster kubelet: I1020 10:11:50.737968 30409 plugin_manager.go:116] Starting Kubelet Plugin Manager
169Oct 20 10:11:50 kubemaster kubelet: I1020 10:11:50.742599 30409 kubelet_node_status.go:286] Setting node annotation to enable volume controller attach/detach
170Oct 20 10:12:01 kubemaster kubelet: I1020 10:12:01.174518 30409 kubelet_node_status.go:286] Setting node annotation to enable volume controller attach/detach
171Oct 20 10:12:11 kubemaster kubelet: I1020 10:12:11.645369 30409 kubelet_node_status.go:286] Setting node annotation to enable volume controller attach/detach
172Oct 20 10:12:22 kubemaster kubelet: I1020 10:12:22.691694 30409 kubelet_node_status.go:286] Setting node annotation to enable volume controller attach/detach
173Oct 20 10:12:33 kubemaster kubelet: I1020 10:12:33.306539 30409 kubelet_node_status.go:286] Setting node annotation to enable volume controller attach/detach
174Oct 20 10:12:44 kubemaster kubelet: I1020 10:12:44.117613 30409 kubelet_node_status.go:286] Setting node annotation to enable volume controller attach/detach