· 6 years ago · Oct 14, 2019, 11:24 PM
1(venv) [wusui@localhost ocs-ci]$ run-ci -m deployment --cluster-conf conf/ocs_basic_install.yml --cluster-name wx-wusui-ocs-cluster-xx --cluster-path /home/wusui/wusui tests/ --deploy
2The kubeconfig file /home/wusui/wusui/auth/kubeconfig doesn't exist!
3============================= test session starts ==============================
4platform linux -- Python 3.6.8, pytest-5.2.1, py-1.8.0, pluggy-0.13.0
5rootdir: /home/wusui/ocs-ci, inifile: pytest.ini
6plugins: metadata-1.8.0, logger-0.5.1, marker-bugzilla-0.9.3, reportportal-1.0.5, html-2.0.0
7collecting ...
8----------------------------- live log collection ------------------------------
915:18:07 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc get Node --selector=node-role.kubernetes.io/worker -o yaml
10collected 205 items / 204 deselected / 1 selected
11
12tests/ecosystem/deployment/test_deployment.py::test_deployment
13-------------------------------- live log setup --------------------------------
1415:18:09 - MainThread - tests.conftest - INFO - All logs located at /tmp/ocs-ci-logs-1571091486
1515:18:09 - MainThread - ocs_ci.deployment.factory - INFO - Deployment key = aws_ipi
1615:18:09 - MainThread - ocs_ci.deployment.factory - INFO - Current deployment platform: aws,deployment type: ipi
1715:18:09 - MainThread - ocs_ci.utility.utils - INFO - Downloading openshift client (4.2.0-0.nightly-2019-10-11-230724).
1815:18:19 - MainThread - ocs_ci.utility.utils - INFO - Executing command: tar xzvf openshift-client.tar.gz oc kubectl
1915:18:19 - MainThread - ocs_ci.utility.utils - INFO - Executing command: /home/wusui/ocs-ci/bin/oc version
2015:18:23 - MainThread - ocs_ci.utility.utils - INFO - OpenShift Client version: Client Version: openshift-clients-4.2.0-201910041700
21 Kubernetes Version: v1.11.0+d4cacc0
2215:18:23 - MainThread - ocs_ci.ocs.openshift_ops - INFO - Testing access to cluster with /home/wusui/wusui/auth/kubeconfig
2315:18:23 - MainThread - ocs_ci.ocs.openshift_ops - WARNING - The kubeconfig file /home/wusui/wusui/auth/kubeconfig doesn't exist!
2415:18:23 - MainThread - botocore.credentials - INFO - Found credentials in shared credentials file: ~/.aws/credentials
2515:18:24 - MainThread - ocs_ci.utility.utils - INFO - Downloading openshift installer (4.2.0-0.nightly-2019-10-11-230724).
2615:18:32 - MainThread - ocs_ci.utility.utils - INFO - Executing command: tar xzvf openshift-install.tar.gz openshift-install
2715:18:33 - MainThread - ocs_ci.utility.utils - INFO - Executing command: /home/wusui/ocs-ci/bin/openshift-install version
2815:18:33 - MainThread - ocs_ci.utility.utils - INFO - OpenShift Installer version: /home/wusui/ocs-ci/bin/openshift-install v4.2.0
29 built from commit 90ccb37ac1f85ae811c50a29f9bb7e779c5045fb
30 release image registry.svc.ci.openshift.org/ocp/release@sha256:d5c059a05eb5fafe0d880a4d962690d3b171d5467ab6fbfccf1fb221f203fbea
3115:18:33 - MainThread - ocs_ci.deployment.ocp - INFO - A testing cluster will be deployed and cluster information stored at: /home/wusui/wusui
3215:18:33 - MainThread - ocs_ci.deployment.ocp - INFO - Generating install-config
3315:18:33 - MainThread - ocs_ci.deployment.ocp - INFO - Install config:
34 apiVersion: v1
35 baseDomain: qe.rh-ocs.com
36 compute:
37 - name: worker
38 platform:
39 aws:
40 type: m4.xlarge
41 replicas: 3
42 controlPlane:
43 name: master
44 platform:
45 aws:
46 type: m4.xlarge
47 replicas: 3
48 metadata:
49 creationTimestamp: null
50 name: 'wx-wusui-ocs-cluster-xx'
51 networking:
52 clusterNetwork:
53 - cidr: 10.128.0.0/14
54 hostPrefix: 23
55 machineCIDR: 10.0.0.0/16
56 networkType: OpenShiftSDN
57 serviceNetwork:
58 - 172.30.0.0/16
59 platform:
60 aws:
61 region: us-east-2
62 pullSecret: ''
6315:18:33 - MainThread - ocs_ci.deployment.aws - INFO - Deploying OCP cluster
6415:18:33 - MainThread - ocs_ci.deployment.aws - INFO - Openshift-installer will be using loglevel:INFO
6515:18:33 - MainThread - ocs_ci.utility.utils - INFO - Executing command: /home/wusui/ocs-ci/bin/openshift-install create cluster --dir /home/wusui/wusui --log-level INFO
6615:47:02 - MainThread - ocs_ci.utility.utils - WARNING - Command warning: level=info msg="Consuming \"Install Config\" from target directory"
67 level=info msg="Creating infrastructure resources..."
68 level=info msg="Waiting up to 30m0s for the Kubernetes API at https://api.wx-wusui-ocs-cluster-xx.qe.rh-ocs.com:6443..."
69 level=info msg="API v1.14.6+1e5e0b1 up"
70 level=info msg="Waiting up to 30m0s for bootstrapping to complete..."
71 level=info msg="Destroying the bootstrap resources..."
72 level=info msg="Waiting up to 30m0s for the cluster at https://api.wx-wusui-ocs-cluster-xx.qe.rh-ocs.com:6443 to initialize..."
73 level=info msg="Waiting up to 10m0s for the openshift-console route to be created..."
74 level=info msg="Install complete!"
75 level=info msg="To access the cluster as the system:admin user when using 'oc', run 'export KUBECONFIG=/home/wusui/wusui/auth/kubeconfig'"
76 level=info msg="Access the OpenShift web-console here: https://console-openshift-console.apps.wx-wusui-ocs-cluster-xx.qe.rh-ocs.com"
77 level=info msg="Login to the console with user: kubeadmin, password: 7C5YE-WXgav-L6iiq-7skuA"
7815:47:02 - MainThread - ocs_ci.ocs.openshift_ops - INFO - Testing access to cluster with /home/wusui/wusui/auth/kubeconfig
7915:47:02 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc cluster-info
8015:47:04 - MainThread - ocs_ci.ocs.openshift_ops - INFO - Access to cluster is OK!
8115:47:04 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get CephCluster -n openshift-storage -o yaml
8215:47:04 - MainThread - ocs_ci.deployment.deployment - INFO - Running OCS basic installation
8315:47:05 - MainThread - ocs_ci.deployment.deployment - INFO - Deployment of OCS via OCS operator
8415:47:07 - MainThread - ocs_ci.utility.templating - INFO - apiVersion: v1
85 kind: Namespace
86 metadata:
87 labels:
88 openshift.io/cluster-monitoring: 'true'
89 name: openshift-storage
90 spec: {}
91 ---
92 apiVersion: v1
93 kind: Namespace
94 metadata:
95 name: local-storage
96 spec: {}
97 ---
98 apiVersion: operators.coreos.com/v1
99 kind: OperatorGroup
100 metadata:
101 name: openshift-storage-operatorgroup
102 namespace: openshift-storage
103 spec:
104 serviceAccount:
105 metadata:
106 creationTimestamp: null
107 targetNamespaces:
108 - openshift-storage
109 ---
110 apiVersion: operators.coreos.com/v1
111 kind: OperatorGroup
112 metadata:
113 name: local-operator-group
114 namespace: local-storage
115 spec:
116 serviceAccount:
117 metadata:
118 creationTimestamp: null
119 targetNamespaces:
120 - openshift-storage
121 ---
122 apiVersion: operators.coreos.com/v1alpha1
123 kind: CatalogSource
124 metadata:
125 name: local-storage-manifests
126 namespace: openshift-marketplace
127 spec:
128 description: An operator to manage local volumes
129 displayName: Local Storage Operator
130 icon:
131 base64data: ''
132 mediatype: ''
133 image: quay.io/gnufied/local-registry:v4.2.0
134 publisher: Red Hat
135 sourceType: grpc
136 ---
137 apiVersion: operators.coreos.com/v1alpha1
138 kind: CatalogSource
139 metadata:
140 name: ocs-catalogsource
141 namespace: openshift-marketplace
142 spec:
143 displayName: Openshift Container Storage
144 icon:
145 base64data: ''
146 mediatype: ''
147 image: quay.io/rhceph-dev/ocs-registry:4.2-139.3c517a0.release_4.2
148 publisher: Red Hat
149 sourceType: grpc
15015:47:07 - MainThread - ocs_ci.utility.templating - INFO - apiVersion: operators.coreos.com/v1alpha1
151 kind: Subscription
152 metadata:
153 name: ocs-subscription
154 namespace: openshift-storage
155 spec:
156 channel: alpha
157 name: ocs-operator
158 source: ocs-catalogsource
159 sourceNamespace: openshift-marketplace
16015:47:07 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc --kubeconfig /home/wusui/wusui/auth/kubeconfig get node -o yaml
16115:47:08 - MainThread - ocs_ci.deployment.deployment - INFO - Label nodes: ip-10-0-133-189.us-east-2.compute.internal ip-10-0-152-15.us-east-2.compute.internal ip-10-0-174-134.us-east-2.compute.internal with label: cluster.ocs.openshift.io/openshift-storage=''
16215:47:08 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc --kubeconfig /home/wusui/wusui/auth/kubeconfig label nodes ip-10-0-133-189.us-east-2.compute.internal ip-10-0-152-15.us-east-2.compute.internal ip-10-0-174-134.us-east-2.compute.internal cluster.ocs.openshift.io/openshift-storage=''
16315:47:09 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc create -f /tmp/olm_manifestnqtdkkdl
16415:47:10 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-marketplace --kubeconfig /home/wusui/wusui/auth/kubeconfig get CatalogSource ocs-catalogsource -n openshift-marketplace -o yaml
16515:47:10 - MainThread - ocs_ci.ocs.resources.catalog_source - INFO - Catalog source ocs-catalogsource is in state: IDLE!
16615:47:15 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-marketplace --kubeconfig /home/wusui/wusui/auth/kubeconfig get CatalogSource ocs-catalogsource -n openshift-marketplace -o yaml
16715:47:16 - MainThread - ocs_ci.ocs.resources.catalog_source - INFO - Catalog source ocs-catalogsource is in state: CONNECTING!
16815:47:21 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-marketplace --kubeconfig /home/wusui/wusui/auth/kubeconfig get CatalogSource ocs-catalogsource -n openshift-marketplace -o yaml
16915:47:21 - MainThread - ocs_ci.ocs.resources.catalog_source - INFO - Catalog source ocs-catalogsource is in state: CONNECTING!
17015:47:26 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-marketplace --kubeconfig /home/wusui/wusui/auth/kubeconfig get CatalogSource ocs-catalogsource -n openshift-marketplace -o yaml
17115:47:27 - MainThread - ocs_ci.ocs.resources.catalog_source - INFO - Catalog source ocs-catalogsource is in state: CONNECTING!
17215:47:32 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-marketplace --kubeconfig /home/wusui/wusui/auth/kubeconfig get CatalogSource ocs-catalogsource -n openshift-marketplace -o yaml
17315:47:33 - MainThread - ocs_ci.ocs.resources.catalog_source - INFO - Catalog source ocs-catalogsource is in state: CONNECTING!
17415:47:38 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-marketplace --kubeconfig /home/wusui/wusui/auth/kubeconfig get CatalogSource ocs-catalogsource -n openshift-marketplace -o yaml
17515:47:38 - MainThread - ocs_ci.ocs.resources.catalog_source - INFO - Catalog source ocs-catalogsource is in state: READY!
17615:47:38 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc create -f /tmp/subscription_manifestlaxgmuua
17715:47:39 - MainThread - ocs_ci.ocs.resources.packagemanifest - INFO - Waiting for a resource(s) of kind packagemanifest identified by name ''
17815:47:39 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-marketplace --kubeconfig /home/wusui/wusui/auth/kubeconfig get packagemanifest ocs-operator -n openshift-marketplace -o yaml
17915:47:42 - MainThread - ocs_ci.ocs.resources.packagemanifest - INFO - package manifest ocs-operator found!
18015:47:42 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-marketplace --kubeconfig /home/wusui/wusui/auth/kubeconfig get packagemanifest ocs-operator -n openshift-marketplace -o yaml
18115:47:45 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
18215:47:46 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Pending!
18315:47:51 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
18415:47:51 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Pending!
18515:47:56 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
18615:47:57 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Pending!
18715:48:02 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
18815:48:03 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Pending!
18915:48:08 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
19015:48:09 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: InstallReady!
19115:48:14 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
19215:48:15 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Installing!
19315:48:20 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
19415:48:21 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Installing!
19515:48:26 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
19615:48:27 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Installing!
19715:48:32 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
19815:48:33 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Installing!
19915:48:38 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
20015:48:39 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Installing!
20115:48:44 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
20215:48:45 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Installing!
20315:48:50 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get csv ocs-operator.v0.0.1 -n openshift-storage -o yaml
20415:48:51 - MainThread - ocs_ci.ocs.resources.csv - INFO - CSV ocs-operator.v0.0.1 is in phase: Succeeded!
20515:48:52 - MainThread - ocs_ci.utility.templating - INFO - apiVersion: ocs.openshift.io/v1
206 kind: StorageCluster
207 metadata:
208 name: rook-ceph
209 namespace: openshift-storage
210 spec:
211 manageNodes: false
212 monPVCTemplate:
213 spec:
214 accessModes:
215 - ReadWriteOnce
216 resources:
217 requests:
218 storage: 10Gi
219 storageClassName: gp2
220 resources:
221 mds:
222 Limits: null
223 Requests: null
224 mgr:
225 Limits: null
226 Requests: null
227 mon:
228 Limits: null
229 Requests: null
230 noobaa-core:
231 Limits: null
232 Requests: null
233 noobaa-db:
234 Limits: null
235 Requests: null
236 rgw:
237 Limits: null
238 Requests: null
239 storageDeviceSets:
240 - count: 3
241 dataPVCTemplate:
242 spec:
243 accessModes:
244 - ReadWriteOnce
245 resources:
246 requests:
247 storage: 100Gi
248 storageClassName: gp2
249 volumeMode: Block
250 name: example-deviceset
251 placement: {}
252 portable: true
253 resources:
254 Limits: null
255 Requests: null
25615:48:52 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc create -f /tmp/cluster_storage09ooxueo
25715:48:53 - MainThread - ocs_ci.ocs.ocp - INFO - Waiting for a resource(s) of kind Pod identified by name '' and selector app=rook-ceph-mon to reach desired condition Running
25815:48:53 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
25915:48:54 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
26015:48:57 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
26115:48:57 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
26215:49:00 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
26315:49:01 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
26415:49:04 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
26515:49:04 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
26615:49:07 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
26715:49:08 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
26815:49:11 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
26915:49:11 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
27015:49:14 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
27115:49:15 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-canary-75c7488d57-ssbvz -n openshift-storage
27215:49:15 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['ContainerCreating'], but we were waiting for all 3 of them to be Running
27315:49:18 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
27415:49:19 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-canary-75c7488d57-ssbvz -n openshift-storage
27515:49:20 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-canary-7bcdcd86d4-9t7dp -n openshift-storage
27615:49:20 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['ContainerCreating', 'ContainerCreating'], but we were waiting for all 3 of them to be Running
27715:49:23 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
27815:49:24 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-canary-75c7488d57-ssbvz -n openshift-storage
27915:49:24 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-canary-7bcdcd86d4-9t7dp -n openshift-storage
28015:49:25 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-canary-669f56854c-rl5l9 -n openshift-storage
28115:49:25 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['ContainerCreating', 'ContainerCreating', 'ContainerCreating'], but we were waiting for all 3 of them to be Running
28215:49:28 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
28315:49:29 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-canary-75c7488d57-ssbvz -n openshift-storage
28415:49:30 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-canary-7bcdcd86d4-9t7dp -n openshift-storage
28515:49:30 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-canary-669f56854c-rl5l9 -n openshift-storage
28615:49:31 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Terminating', 'ContainerCreating', 'ContainerCreating'], but we were waiting for all 3 of them to be Running
28715:49:34 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
28815:49:34 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-canary-75c7488d57-ssbvz -n openshift-storage
28915:49:35 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-canary-7bcdcd86d4-9t7dp -n openshift-storage
29015:49:35 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-canary-669f56854c-rl5l9 -n openshift-storage
29115:49:36 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Terminating', 'ContainerCreating', 'ContainerCreating'], but we were waiting for all 3 of them to be Running
29215:49:39 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
29315:49:40 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-canary-7bcdcd86d4-9t7dp -n openshift-storage
29415:49:40 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-canary-669f56854c-rl5l9 -n openshift-storage
29515:49:41 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['ContainerCreating', 'ContainerCreating'], but we were waiting for all 3 of them to be Running
29615:49:44 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
29715:49:44 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-canary-669f56854c-rl5l9 -n openshift-storage
29815:49:45 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Terminating'], but we were waiting for all 3 of them to be Running
29915:49:48 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
30015:49:48 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-canary-669f56854c-rl5l9 -n openshift-storage
30115:49:49 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Terminating'], but we were waiting for all 3 of them to be Running
30215:49:52 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
30315:49:52 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
30415:49:55 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
30515:49:56 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
30615:49:57 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Init:0/2'], but we were waiting for all 3 of them to be Running
30715:50:00 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
30815:50:00 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
30915:50:01 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Init:0/2'], but we were waiting for all 3 of them to be Running
31015:50:04 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
31115:50:04 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
31215:50:05 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Init:0/2'], but we were waiting for all 3 of them to be Running
31315:50:08 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
31415:50:09 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
31515:50:09 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Init:0/2'], but we were waiting for all 3 of them to be Running
31615:50:12 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
31715:50:13 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
31815:50:13 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['PodInitializing'], but we were waiting for all 3 of them to be Running
31915:50:16 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
32015:50:17 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
32115:50:17 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running'], but we were waiting for all 3 of them to be Running
32215:50:20 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
32315:50:21 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
32415:50:22 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
32515:50:22 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Init:0/2'], but we were waiting for all 3 of them to be Running
32615:50:25 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
32715:50:26 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
32815:50:27 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
32915:50:27 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Init:0/2'], but we were waiting for all 3 of them to be Running
33015:50:30 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
33115:50:31 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
33215:50:31 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
33315:50:32 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Init:0/2'], but we were waiting for all 3 of them to be Running
33415:50:35 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
33515:50:36 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
33615:50:36 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
33715:50:37 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Init:0/2'], but we were waiting for all 3 of them to be Running
33815:50:40 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
33915:50:40 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
34015:50:41 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
34115:50:41 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'PodInitializing'], but we were waiting for all 3 of them to be Running
34215:50:44 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
34315:50:45 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
34415:50:46 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
34515:50:46 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Running'], but we were waiting for all 3 of them to be Running
34615:50:49 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
34715:50:50 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
34815:50:50 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
34915:50:51 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Running'], but we were waiting for all 3 of them to be Running
35015:50:54 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
35115:50:55 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
35215:50:55 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
35315:50:56 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-75c574c66b-zzjmg -n openshift-storage
35415:50:56 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Running', 'Init:0/2'], but we were waiting for all 3 of them to be Running
35515:50:59 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
35615:51:00 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
35715:51:01 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
35815:51:01 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-75c574c66b-zzjmg -n openshift-storage
35915:51:02 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Running', 'Init:0/2'], but we were waiting for all 3 of them to be Running
36015:51:05 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
36115:51:05 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
36215:51:06 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
36315:51:06 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-75c574c66b-zzjmg -n openshift-storage
36415:51:07 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Running', 'Init:0/2'], but we were waiting for all 3 of them to be Running
36515:51:10 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
36615:51:11 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
36715:51:11 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
36815:51:12 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-75c574c66b-zzjmg -n openshift-storage
36915:51:12 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Running', 'PodInitializing'], but we were waiting for all 3 of them to be Running
37015:51:15 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
37115:51:16 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage
37215:51:17 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage
37315:51:17 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-75c574c66b-zzjmg -n openshift-storage
37415:51:18 - MainThread - ocs_ci.ocs.ocp - INFO - Waiting for a resource(s) of kind Pod identified by name '' and selector app=rook-ceph-mgr to reach desired condition Running
37515:51:18 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mgr -o yaml
37615:51:18 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all of them to be Running
37715:51:21 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mgr -o yaml
37815:51:22 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all of them to be Running
37915:51:25 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mgr -o yaml
38015:51:25 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mgr-a-6797774c9-b7tfs -n openshift-storage
38115:51:26 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['ContainerCreating'], but we were waiting for all of them to be Running
38215:51:29 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mgr -o yaml
38315:51:29 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mgr-a-6797774c9-b7tfs -n openshift-storage
38415:51:30 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['ContainerCreating'], but we were waiting for all of them to be Running
38515:51:33 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mgr -o yaml
38615:51:33 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mgr-a-6797774c9-b7tfs -n openshift-storage
38715:51:34 - MainThread - ocs_ci.ocs.ocp - INFO - Waiting for a resource(s) of kind Pod identified by name '' and selector app=rook-ceph-osd to reach desired condition Running
38815:51:34 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
38915:51:34 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
39015:51:37 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
39115:51:38 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
39215:51:41 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
39315:51:42 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
39415:51:45 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
39515:51:45 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
39615:51:48 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
39715:51:49 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
39815:51:52 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
39915:51:52 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
40015:51:55 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
40115:51:56 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
40215:51:59 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
40315:51:59 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
40415:52:02 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
40515:52:03 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
40615:52:06 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
40715:52:06 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were [], but we were waiting for all 3 of them to be Running
40815:52:09 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
40915:52:10 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-0-bb8bd696c-kd4gs -n openshift-storage
41015:52:11 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Pending'], but we were waiting for all 3 of them to be Running
41115:52:14 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
41215:52:14 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-0-bb8bd696c-kd4gs -n openshift-storage
41315:52:15 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Init:0/3'], but we were waiting for all 3 of them to be Running
41415:52:18 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
41515:52:18 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-0-bb8bd696c-kd4gs -n openshift-storage
41615:52:19 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-1-7b57b8dffb-88lnx -n openshift-storage
41715:52:20 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Init:0/3', 'Init:0/3'], but we were waiting for all 3 of them to be Running
41815:52:23 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
41915:52:23 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-0-bb8bd696c-kd4gs -n openshift-storage
42015:52:24 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-1-7b57b8dffb-88lnx -n openshift-storage
42115:52:24 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-2-d59fb676-6slhn -n openshift-storage
42215:52:25 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Init:0/3', 'Init:0/3', 'Init:0/3'], but we were waiting for all 3 of them to be Running
42315:52:28 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
42415:52:29 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-0-bb8bd696c-kd4gs -n openshift-storage
42515:52:29 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-1-7b57b8dffb-88lnx -n openshift-storage
42615:52:30 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-2-d59fb676-6slhn -n openshift-storage
42715:52:30 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Init:2/3', 'Init:0/3', 'Init:0/3'], but we were waiting for all 3 of them to be Running
42815:52:33 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
42915:52:34 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-0-bb8bd696c-kd4gs -n openshift-storage
43015:52:35 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-1-7b57b8dffb-88lnx -n openshift-storage
43115:52:35 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-2-d59fb676-6slhn -n openshift-storage
43215:52:36 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'Init:0/3', 'Init:1/3'], but we were waiting for all 3 of them to be Running
43315:52:39 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
43415:52:39 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-0-bb8bd696c-kd4gs -n openshift-storage
43515:52:40 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-1-7b57b8dffb-88lnx -n openshift-storage
43615:52:40 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-2-d59fb676-6slhn -n openshift-storage
43715:52:41 - MainThread - ocs_ci.ocs.ocp - INFO - status of item(s) were ['Running', 'PodInitializing', 'PodInitializing'], but we were waiting for all 3 of them to be Running
43815:52:44 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
43915:52:45 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-0-bb8bd696c-kd4gs -n openshift-storage
44015:52:45 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-1-7b57b8dffb-88lnx -n openshift-storage
44115:52:46 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-2-d59fb676-6slhn -n openshift-storage
44215:52:46 - MainThread - ocs_ci.deployment.deployment - INFO - Validate mon and OSD are backed by PVCs
44315:52:46 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get PersistentVolumeClaim -n openshift-storage --selector=app=rook-ceph-mon -o yaml
44415:52:54 - MainThread - ocs_ci.ocs.cluster - INFO - PVC rook-ceph-mon-a is in Bound state
44515:52:54 - MainThread - ocs_ci.ocs.cluster - INFO - PVC rook-ceph-mon-b is in Bound state
44615:52:54 - MainThread - ocs_ci.ocs.cluster - INFO - PVC rook-ceph-mon-c is in Bound state
44715:52:54 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-mon -o yaml
44815:52:54 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-a-65648675db-gbffh -n openshift-storage -o yaml
44915:52:55 - MainThread - ocs_ci.ocs.cluster - INFO - OCS pod rook-ceph-mon-a-65648675db-gbffh is backed by PVC rook-ceph-mon-a
45015:52:55 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig rsh rook-ceph-mon-a-65648675db-gbffh df -kh
45115:52:56 - MainThread - ocs_ci.ocs.cluster - INFO - PVC rook-ceph-mon-a is mounted on pod rook-ceph-mon-a-65648675db-gbffh
45215:52:56 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-b-7747b9fb4f-5lkk5 -n openshift-storage -o yaml
45315:52:57 - MainThread - ocs_ci.ocs.cluster - INFO - OCS pod rook-ceph-mon-b-7747b9fb4f-5lkk5 is backed by PVC rook-ceph-mon-b
45415:52:57 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig rsh rook-ceph-mon-b-7747b9fb4f-5lkk5 df -kh
45515:52:59 - MainThread - ocs_ci.ocs.cluster - INFO - PVC rook-ceph-mon-b is mounted on pod rook-ceph-mon-b-7747b9fb4f-5lkk5
45615:52:59 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-mon-c-75c574c66b-zzjmg -n openshift-storage -o yaml
45715:52:59 - MainThread - ocs_ci.ocs.cluster - INFO - OCS pod rook-ceph-mon-c-75c574c66b-zzjmg is backed by PVC rook-ceph-mon-c
45815:52:59 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig rsh rook-ceph-mon-c-75c574c66b-zzjmg df -kh
45915:53:01 - MainThread - ocs_ci.ocs.cluster - INFO - PVC rook-ceph-mon-c is mounted on pod rook-ceph-mon-c-75c574c66b-zzjmg
46015:53:01 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get PersistentVolumeClaim -n openshift-storage --selector=ceph.rook.io/DeviceSet=example-deviceset -o yaml
46115:53:01 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod -n openshift-storage --selector=app=rook-ceph-osd -o yaml
46215:53:02 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-0-bb8bd696c-kd4gs -n openshift-storage -o yaml
46315:53:02 - MainThread - ocs_ci.ocs.cluster - ERROR - rook-ceph-osd-0-bb8bd696c-kd4gs is not backed by designated PVC
46415:53:02 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-1-7b57b8dffb-88lnx -n openshift-storage -o yaml
46515:53:03 - MainThread - ocs_ci.ocs.cluster - ERROR - rook-ceph-osd-1-7b57b8dffb-88lnx is not backed by designated PVC
46615:53:03 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc -n openshift-storage --kubeconfig /home/wusui/wusui/auth/kubeconfig get Pod rook-ceph-osd-2-d59fb676-6slhn -n openshift-storage -o yaml
46715:53:04 - MainThread - ocs_ci.ocs.cluster - ERROR - rook-ceph-osd-2-d59fb676-6slhn is not backed by designated PVC
46815:53:04 - MainThread - ocs_ci.ocs.utils - INFO - Must gather image: quay.io/openshift/origin-must-gather will be used.
46915:53:04 - MainThread - ocs_ci.ocs.utils - INFO - OCS logs will be placed in location /tmp/failed_testcase_ocs_logs_1571091486/deployment_ocs_logs/ocp_must_gather
47015:53:04 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc --kubeconfig /home/wusui/wusui/auth/kubeconfig adm must-gather --image=quay.io/openshift/origin-must-gather --dest-dir=/tmp/failed_testcase_ocs_logs_1571091486/deployment_ocs_logs/ocp_must_gather
47115:57:55 - MainThread - ocs_ci.ocs.utils - INFO - Must gather image: quay.io/openshift/origin-must-gather will be used.
47215:57:55 - MainThread - ocs_ci.ocs.utils - INFO - OCS logs will be placed in location /tmp/failed_testcase_ocs_logs_1571091486/deployment_ocs_logs/ocp_must_gather
47315:57:55 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc --kubeconfig /home/wusui/wusui/auth/kubeconfig adm must-gather --image=quay.io/openshift/origin-must-gather --dest-dir=/tmp/failed_testcase_ocs_logs_1571091486/deployment_ocs_logs/ocp_must_gather -- /usr/bin/gather_service_logs worker
47415:58:15 - MainThread - ocs_ci.ocs.utils - INFO - Must gather image: quay.io/rhceph-dev/ocs-must-gather will be used.
47515:58:15 - MainThread - ocs_ci.ocs.utils - INFO - OCS logs will be placed in location /tmp/failed_testcase_ocs_logs_1571091486/deployment_ocs_logs/ocs_must_gather
47615:58:15 - MainThread - ocs_ci.utility.utils - INFO - Executing command: oc --kubeconfig /home/wusui/wusui/auth/kubeconfig adm must-gather --image=quay.io/rhceph-dev/ocs-must-gather --dest-dir=/tmp/failed_testcase_ocs_logs_1571091486/deployment_ocs_logs/ocs_must_gather
47716:00:39 - MainThread - ocs_ci.utility.utils - WARNING - Command warning: WARNING: cannot use rsync: rsync not available in container
478ERROR [100%]
479
480==================================== ERRORS ====================================
481______________________ ERROR at setup of test_deployment _______________________
482
483request = <SubRequest 'cluster' for <Function test_deployment>>
484log_cli_level = 'INFO'
485
486 @pytest.fixture(scope="session", autouse=True)
487 def cluster(request, log_cli_level):
488 """
489 This fixture initiates deployment for both OCP and OCS clusters.
490 Specific platform deployment classes will handle the fine details
491 of action
492 """
493 log.info(f"All logs located at {ocsci_log_path()}")
494
495 teardown = config.RUN['cli_params']['teardown']
496 deploy = config.RUN['cli_params']['deploy']
497 factory = dep_factory.DeploymentFactory()
498 deployer = factory.get_deployment()
499
500 # Add a finalizer to teardown the cluster after test execution is finished
501 if teardown:
502 def cluster_teardown_finalizer():
503 deployer.destroy_cluster(log_cli_level)
504 request.addfinalizer(cluster_teardown_finalizer)
505 log.info("Will teardown cluster because --teardown was provided")
506
507 # Download client
508 force_download = (
509 config.RUN['cli_params'].get('deploy')
510 and config.DEPLOYMENT['force_download_client']
511 )
512 get_openshift_client(force_download=force_download)
513
514 if deploy:
515 # Deploy cluster
516> deployer.deploy_cluster(log_cli_level)
517
518tests/conftest.py:712:
519_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
520ocs_ci/deployment/deployment.py:90: in deploy_cluster
521 self.deploy_ocs()
522ocs_ci/deployment/deployment.py:390: in deploy_ocs
523 validate_cluster_on_pvc(label=constants.DEFAULT_DEVICESET_LABEL)
524_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
525
526label = 'ceph.rook.io/DeviceSet=example-deviceset'
527
528 def validate_cluster_on_pvc(label):
529 """
530 Validate creation of PVCs for MON and OSD pods.
531 Also validate that those PVCs are attached to the OCS pods
532
533 Args:
534 label(string): Label for MON or OSD PVCs
535
536 Raises:
537 AssertionError: If PVC is not mounted on one or more OCS pods
538
539 """
540 # Get the PVCs for selected label (MON/OSD)
541 ocs_pvc_obj = get_all_pvc_objs(
542 namespace=defaults.ROOK_CLUSTER_NAMESPACE, selector=label
543 )
544
545 # Check all pvc's are in bound state
546 for pvc_obj in ocs_pvc_obj:
547 assert pvc_obj.status == constants.STATUS_BOUND, (
548 f"PVC {pvc_obj.name} is not Bound"
549 )
550 logger.info(f"PVC {pvc_obj.name} is in Bound state")
551
552 # Get OCS pod names based on selected label
553 if label == constants.MON_APP_LABEL:
554 ocs_pod_obj = pod.get_mon_pods(
555 mon_label=label, namespace=defaults.ROOK_CLUSTER_NAMESPACE
556 )
557 if label == constants.DEFAULT_DEVICESET_LABEL:
558 ocs_pod_obj = pod.get_osd_pods(
559 osd_label=constants.OSD_APP_LABEL,
560 namespace=defaults.ROOK_CLUSTER_NAMESPACE
561 )
562
563 # Create a pvc list for requested label
564 pvc_list = []
565 for pvc_obj in ocs_pvc_obj:
566 pvc_list.append(pvc_obj.name)
567
568 # Check if PVC is mounted on designated OCS pod
569 # loop over all PVCs in the pvc_list
570 # Each mon and osd pod is expected to have only one Claim attached
571 claim_found = True
572 all_backed_by_pvc = True
573 for pod_obj in ocs_pod_obj:
574 pod_volumes = pod_obj.get().get('spec').get('volumes')
575 claim_spec_exists = False
576 for volumes in pod_volumes:
577 pvc = volumes.get('persistentVolumeClaim')
578 if pvc:
579 claim_name = pvc.get('claimName')
580 claim_spec_exists = True
581 backed_by_pvc = False
582 if claim_name in pvc_list:
583 logger.info(
584 f"OCS pod {pod_obj.name} is backed by PVC {claim_name}"
585 )
586 # If backed by PVC, set backed_by_pvc = True
587 backed_by_pvc = True
588
589 # Check if Mon PVC is mounted as /var/lib/ceph/mon/ceph-x
590 if label == constants.MON_APP_LABEL:
591 mount_point = pod_obj.exec_cmd_on_pod(command="df -kh")
592 assert "/var/lib/ceph/mon/ceph" in mount_point, (
593 f"pvc is not mounted on pod {pod_obj.name}"
594 )
595 logger.info(
596 f"PVC {claim_name} is mounted"
597 f" on pod {pod_obj.name}"
598 )
599 if not backed_by_pvc:
600 logger.error(f"{pod_obj.name} is not backed by designated PVC ")
601 all_backed_by_pvc = False
602
603 # If no PVC is mounted, print error and continue checking other pods
604 if not claim_spec_exists:
605 logger.error(
606 f"No PersistentVolumeClaim spec found in "
607 f"OCS pod {pod_obj.name}"
608 )
609 claim_found = False
610
611 # Even if one OCS POD is not backed by a PVC, fail the deployment
612 assert all_backed_by_pvc, (
613> "One or more pods are not backed by a PVC "
614 "please check deployment logs"
615 )
616E AssertionError: One or more pods are not backed by a PVC please check deployment logs
617
618ocs_ci/ocs/cluster.py:514: AssertionError
619================ 204 deselected, 1 error in 2552.28s (0:42:32) =================
620sys:1: ResourceWarning: unclosed file <_io.TextIOWrapper name=16 mode='w+' encoding='UTF-8'>
621sys:1: ResourceWarning: unclosed file <_io.TextIOWrapper name=15 mode='w+' encoding='UTF-8'>
622sys:1: ResourceWarning: unclosed file <_io.TextIOWrapper name=14 mode='w+' encoding='UTF-8'>
623sys:1: ResourceWarning: unclosed <ssl.SSLSocket fd=12, family=AddressFamily.AF_INET, type=2049, proto=6, laddr=('192.168.1.5', 46888), raddr=('52.95.18.3', 443)>
624(venv) [wusui@localhost ocs-ci]$ git branch
625* master
626(venv) [wusui@localhost ocs-ci]$ ls tests/e2e/
627__init__.py test_pgsql_pod_respin.py
628logging test_pgsql.py
629monitoring test_pvc_creation_performance.py
630__pycache__ test_pv_scale_and_respin_ceph_pods.py
631test_io_performance.py test_pv_scale_ocs_create_delete_pvcs.py
632test_jenkins_simulation.py test_small_file_workload.py
633(venv) [wusui@localhost ocs-ci]$