· 5 years ago · Nov 20, 2019, 06:24 PM
1/**
2 * pipeline for kaas mirantis demo deployment
3 * CREDENTIALS_ID - gerrit credentials id
4 * GERRIT_HOST - Gerrit host
5 * KAAS_PIPELINE_REFSPEC - Refspec for kaas related pipelines, for testing additional pipeline changes, generally using GERRIT_REFSPEC
6 * GERRIT_REFSPEC - Parameter also used for fetching groovy pipeline from kaas repo [KAAS_PIPELINE_REFSPEC],
7 * default value used in case of non-triggered run, otherwise gerrit trigger will overwrite it
8 *
9 * SLACK_CHANNEL_NOTIFY - If not empty, send build results in slack channel [nightly/git release run]
10 *
11 * kaasImageRepo - KaaS docker repo where kaas related images stored
12 * kaasAwsAccountId - aws account id for ci runs
13 * kaasAwsCredentials - name of jenkins secret with AWSAccessKeyId/AWSSecretKey credentials
14**/
15
16common = new com.mirantis.mk.Common()
17def jUtils = new com.mirantis.mk.JenkinsUtils()
18def gerrit = new com.mirantis.mk.Gerrit()
19jobCatchedErrors = 'No build errors'
20// Artifact repo management
21kaasImageRepo = 'docker-dev-kaas-local.docker.mirantis.net/core'
22
23kaasAwsAccountId = '763213880647'
24kaasAwsCredentials = 'aws-kaas-core-ci-763213880647'
25
26// TODO generate bootstrap creds each time and remove this ones
27bootstrapKaasAwsAccessKeyId = 'AKIA3DMYXTVDWC7ZZWVI'
28bootstrapKaasAwsSecretKey = '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'
29
30// TODO refactor region forwrding after https://mirantis.jira.com/browse/PRODX-1438
31awsDeploymentRegion = 'us-east-2'
32
33def goInfraImage = 'docker-dev-kaas-local.docker.mirantis.net/core/test/backend'
34def goInfraImageTag = 'latest'
35
36def uiImage = 'frontend'
37def awsClusterApiControllerImage = 'aws-cluster-api-controller'
38def iamControllerImage = 'iam-controller'
39def releaseControllerImage = 'release-controller'
40def squidProxyImage = 'squid-cache'
41// Use custom kind image with controllers cache to speedup deploys
42def kindCacheImage = 'kind-cache'
43
44// Nightly/git release run
45// GERRIT_REFSPEC parameter also used for fetching groovy pipeline from kaas repo. 'mirantis' value used in case of non-triggered run
46def headBuild = (env.GERRIT_REFSPEC == 'mirantis' || env.GERRIT_EVENT_TYPE == 'ref-updated')
47
48// Container Deployment parameters
49def buildIdPostfix = (headBuild) ? 'head' : "cr-${env.GERRIT_CHANGE_NUMBER}-${env.GERRIT_PATCHSET_NUMBER}"
50def buildId = "build-${env.BUILD_NUMBER}-${buildIdPostfix}"
51baseContainerName = "goInfra-${buildId}"
52
53node('docker') {
54
55 // Define Jenkins user id's for docker operations
56 def jenkinsUID = common.getJenkinsUid()
57 def jenkinsGID = common.getJenkinsGid()
58 jenkinsUser = "${jenkinsUID}:${jenkinsGID}"
59
60 def workspace = common.getWorkspace()
61 def artifactDir = 'buildArtifacts'
62
63 // Check if dependency jobs passed successfully
64 depsResult = jUtils.checkDependencyJobs()
65 if (!depsResult.status){
66 return
67 }
68
69 //avoid root-owned docker artifacts before kaasLibrary scm
70 sh(script: """\
71 docker run --rm \
72 -v ${workspace}:/code \
73 busybox sh -c 'chown -R ${jenkinsUser} /code'
74 """
75 )
76
77 // Import external kaas groovy library
78 stage('Getting kaas-libraries') {
79 checkout scm
80 baseKaas = load 'hack/backend/kaasLibrary.groovy'
81 }
82
83 /* [KaaS Management cluster bootstrap variables]
84 * Below variables needed for kaas management cluster deployment
85 */
86 kaasClusterName = "kaas-${buildId}"
87
88 // KIND bootstrap definitions
89 kindBootstrapClusterName = "bootstrap-${kaasClusterName}"
90
91 try {
92 timeout(time: 4, unit: 'HOURS') {
93 stage('Checkout') {
94
95 if (headBuild) {
96 common.warningMsg('Product code and groovy pipeline fetched from master branch [mirantis] in kaas/cluster-api-provider-openstack repo')
97 gerrit.gerritPatchsetCheckout('ssh://mcp-jenkins@gerrit.mcp.mirantis.net:29418/kaas/cluster-api-provider-openstack' , 'mirantis', 'HEAD', CREDENTIALS_ID)
98 } else {
99 common.warningMsg("Gerrit-triggered run. Product code and groovy pipeline fetched from ${GERRIT_REFSPEC} in kaas/cluster-api-provider-openstack repo")
100 gerrit.gerritPatchsetCheckout([
101 credentialsId: CREDENTIALS_ID,
102 withWipeOut: true,
103 ])
104 }
105
106 common.infoMsg('Generate valid tags for kaas artifacts')
107 kaasVersion = sh(script: "make DEST=${workspace} GOOS=linux version", returnStdout: true).trim()
108
109 // If we have CR related docker image build from parent job, need to get its tag and use it for our test, otherwise we should use latest tag TODO add additional checkers (ref-updated, etc.) after implementing demo-job inside release-mgmt process
110 if ('kaas_testing_cluster_api_provider_openstack_build_infra' in depsResult.jobs){
111 common.infoMsg('Use infra image from parent job-infra-build..')
112 goInfraImageTag = kaasVersion
113 }
114
115 common.warningMsg("""Make sure that following images exist in artifactory, otherwise kaas deployment will fail.
116 ${kaasImageRepo}/${awsClusterApiControllerImage}:${kaasVersion}
117 ${kaasImageRepo}/${releaseControllerImage}:${kaasVersion}
118 ${kaasImageRepo}/${iamControllerImage}:${kaasVersion}
119 ${kaasImageRepo}/${uiImage}:${kaasVersion}
120 ${kaasImageRepo}/${squidProxyImage}:${kaasVersion}
121 ${kaasImageRepo}/${kindCacheImage}:${kaasVersion}
122 ${goInfraImage}:${goInfraImageTag}""")
123 def customImages = [squidProxyImage, kindCacheImage]
124 common.warningMsg("""[PAY ATTENTION] ${customImages} images may be retagged from latest image w/o rebuild.
125 To check, take a look if job kaas-testing-cluster-api-provider-openstack-build-squid-cache exist in dependencies,
126 and on logs in dependent kaas-testing-cluster-api-provider-openstack-make job""")
127 }
128
129 stage('Download bootstrap tarball') {
130 tarballBootstrapDir = 'bootstrap-from-tarball'
131 sh("mkdir -p ${workspace}/${tarballBootstrapDir}/${artifactDir}")
132
133 def binUrl = sh(script: "make DEST=${workspace} ART_REPO=dev GOOS=linux tarball-url", returnStdout: true).trim()
134 sh("wget -nv ${binUrl} -O bootstrap.tar.gz")
135 sh("tar -C ${tarballBootstrapDir} -zxvf bootstrap.tar.gz")
136 }
137
138 stage('Set KaaS license') {
139 sh("cp hack/mirantis-kaas.lic ${workspace}/${tarballBootstrapDir}")
140 }
141
142 stage('Configure cloud credentials for cluster') {
143 // TODO define auth method for aws
144 def machineTemplatePath = "${workspace}/${tarballBootstrapDir}/templates/aws/machines.yaml.template"
145 def clusterTemplatePath = "${workspace}/${tarballBootstrapDir}/templates/aws/cluster.yaml.template"
146 // TODO add caseswitch in case of many test-dependency conditions
147 def ciInstanceType = 'c5d.large'
148
149 common.infoMsg('Configure cluster and machine templates')
150 machineConf = readYaml file: machineTemplatePath
151 clusterConf = readYaml file: clusterTemplatePath
152 clusterConf['metadata']['name'] = kaasClusterName
153 clusterConf['spec']['providerSpec']['value']['region'] = awsDeploymentRegion
154
155
156 machineConf['items'].eachWithIndex { machine, index ->
157 // max node name = 63 symbols, otherwise vm hostname will be different with machine name in k8s metadata
158 machine['metadata']['name'] = "kaas-demo-mgmt-${index}-${buildId}"
159 machine['spec']['providerSpec']['value']['instanceType'] = ciInstanceType
160 }
161
162 // writeYaml cant override files, need to remove source
163 sh("rm ${machineTemplatePath} ${clusterTemplatePath}")
164 writeYaml file: machineTemplatePath, data: machineConf
165 writeYaml file: clusterTemplatePath, data: clusterConf
166 common.infoMsg('Machine configuration for kaas mgmt generated.')
167 sh("cat ${machineTemplatePath}")
168 }
169
170 stage('Start container with bootstrap') {
171 common.infoMsg('Force update docker image...')
172 sh("docker pull ${goInfraImage}:${goInfraImageTag}")
173
174 // Forward docker socket for KIND bootstrap
175 // TODO return to permanent aws credentials
176 /*
177 withCredentials([
178 [$class: 'UsernamePasswordMultiBinding',
179 credentialsId: kaasAwsCredentials,
180 passwordVariable: 'SECRET_KEY',
181 usernameVariable: 'SECRET_ID'],
182 ],) {
183
184 run docker container here
185 -e AWS_SECRET_ACCESS_KEY=${SECRET_KEY} \
186 -e AWS_ACCESS_KEY_ID=${SECRET_ID} \
187 }
188 */
189 common.warningMsg('TEMP-SOLUTION: reusing bootstrap user credentials')
190 sh(script: """\
191 docker run -d \
192 --name ${baseContainerName} \
193 -v ${workspace}/${tarballBootstrapDir}/:/bootstrap \
194 -v /var/run/docker.sock:/var/run/docker.sock \
195 --network=host \
196 -w=/bootstrap \
197 -e BOOTSTRAP_CLUSTER_NAME=${kindBootstrapClusterName} \
198 -e CLUSTER_NAME=${kaasClusterName} \
199 -e KAAS_AWS_ENABLED=true \
200 -e AWS_SECRET_ACCESS_KEY=${bootstrapKaasAwsSecretKey} \
201 -e AWS_ACCESS_KEY_ID=${bootstrapKaasAwsAccessKeyId} \
202 -e AWS_DEFAULT_REGION=${awsDeploymentRegion} \
203 --entrypoint /usr/bin/tail \
204 ${goInfraImage}:${goInfraImageTag} -f /dev/null
205 """)
206 }
207
208 stage('Configure IAM via CloudFormation') {
209 baseKaas.dockerExecuteCmd(baseContainerName, './kaas bootstrap aws policy')
210 }
211
212 stage('Deploy kaas management cluster') {
213 baseKaas.dockerExecuteCmd(baseContainerName, './bootstrap.sh all')
214 }
215 }
216 currentBuild.result = 'SUCCESS'
217 } catch (e) {
218 // If there was an error or exception thrown, the build failed
219 currentBuild.result = 'FAILURE'
220 jobCatchedErrors = e.message ?: 'Failed to get error msg'
221 throw e
222 } finally {
223 currentBuild.description = """
224 <p>
225 Deployment images:<br/>
226 <b>${awsClusterApiControllerImage} image</b>: ${kaasImageRepo}/${awsClusterApiControllerImage}:${kaasVersion}<br/>
227 <b>${releaseControllerImage} image</b>: ${kaasImageRepo}/${releaseControllerImage}:${kaasVersion}<br/>
228 <b>${iamControllerImage} image</b>: ${kaasImageRepo}/${iamControllerImage}:${kaasVersion}<br/>
229 <b>${uiImage} image</b>: ${kaasImageRepo}/${uiImage}:${kaasVersion}<br/>
230 <b>${squidProxyImage} image</b>: ${kaasImageRepo}/${squidProxyImage}:${kaasVersion}<br/>
231 <b>${kindCacheImage} image</b>: ${kaasImageRepo}/${kindCacheImage}:${kaasVersion}<br/>
232 <b>Infra image</b>: ${goInfraImage}:${goInfraImageTag}<br/>
233 <b>Errors</b>: ${jobCatchedErrors}<br/>
234 </p>
235 """
236
237 stage('Kaas cleanup procedure') {
238 try {
239 // TODO collect specific cluster metadata to verify kaas cleanup
240 timeout(time: 30, unit: 'MINUTES') {
241 //Using kind bootstrap for cleanup ops
242 baseKaas.collectKaasLogs(artifactDir, baseContainerName)
243 common.infoMsg('Cleanup KaaS mgmt & bootstrap')
244 baseKaas.dockerExecuteCmd(baseContainerName, './bootstrap.sh cleanup')
245 }
246 common.infoMsg('TODO Check that resources does not exist after kaas cleanup procedure')
247 // TODO run aws-cleanup in dry-run mode and collect resources to ensure
248 } catch (e) {
249 currentBuild.result = 'FAILURE'
250 baseKaas.collectKaasLogs("${artifactDir}/kaas-logs-after-full-cleanup-failure_${new Date().format('yyyyMMdd_HH-mm-ss').toString()}", baseContainerName)
251 common.errorMsg("Kaas cleanup verification procedure FAILED. \n${e}")
252 }
253 }
254
255 stage('Collect artifacts for debug') {
256 //avoid root-owned docker artifacts
257 baseKaas.setupJenkinsPermissions(jenkinsUser)
258 // Archive BE artifacts
259 archiveArtifacts(
260 artifacts: "${tarballBootstrapDir}/${artifactDir}/**/*.*",
261 allowEmptyArchive: true
262 )
263 }
264
265 if (SLACK_CHANNEL_NOTIFY) {
266 def slack = new com.mirantis.mcp.SlackNotification()
267 slack.jobResultNotification(currentBuild.result, SLACK_CHANNEL_NOTIFY, '', null, '', 'slack_webhook_url')
268 }
269
270 stage('Infra resource cleanup'){
271 common.infoMsg('Trigger aws resource cleanup job')
272 // TODO turn off wait after concurrency enabling
273 // TODO add global region
274 build job: 'kaas-infra-aws-account-cleanup-core-ci', propagate: true, wait: true, parameters: [
275 [$class: 'StringParameterValue', name: 'KAAS_AWS_ACCOUNT_ID', value: kaasAwsAccountId],
276 [$class: 'StringParameterValue', name: 'KAAS_AWS_CREDENTIALS', value: kaasAwsCredentials],
277 [$class: 'StringParameterValue', name: 'KAAS_AWS_REGIONS', value: awsDeploymentRegion],
278 [$class: 'StringParameterValue', name: 'KAAS_PIPELINE_REFSPEC', value: KAAS_PIPELINE_REFSPEC],
279 [$class: 'BooleanParameterValue', name: 'DRY_RUN', value: false]
280 ]
281 // stop ops-container and bootstrap
282 // Kind generated containter name by following scheme for now
283 sh("docker rm -f ${baseContainerName} || true")
284 // remove docker images
285 baseKaas.dockerRmi("""${kaasImageRepo}/${uiImage}:${kaasVersion} \
286 ${kaasImageRepo}/${awsClusterApiControllerImage}:${kaasVersion} \
287 ${kaasImageRepo}/${releaseControllerImage}:${kaasVersion} \
288 ${kaasImageRepo}/${squidProxyImage}:${kaasVersion} \
289 ${kaasImageRepo}/${kindCacheImage}:${kaasVersion} \
290 ${kaasImageRepo}/${iamControllerImage}:${kaasVersion}""", false)
291 // cleanup workspace
292 deleteDir()
293 }
294 }
295}