· 9 years ago · Nov 09, 2016, 03:36 PM
1# -*- coding: utf-8 -*-
2from __future__ import with_statement
3import getpass
4from fabric.api import run, env, hosts, local, cd, task, sudo, settings, runs_once, execute
5from fabric.contrib.files import exists
6from fabric.utils import abort
7import requests
8from datetime import datetime
9import time
10import fpformat
11
12development_host = 'phme-dev.cloudapp.net'
13git_dir = '/home/phme/pichit.me/phme_django'
14git_dir_new = '/home/phme/pichit.me/phme_faraday'
15dir_task_pane_app = '/home/phme/pichit.me/phme_faraday/static/O365/task_pane_app'
16dir_cmp_app = '/home/phme/pichit.me/phme_faraday/static/cmp'
17git_dir_new_pickit = '/home/pickit/pichit.me/phme_faraday'
18
19web_servers = ['phme@phme-web.cloudapp.net',
20 'phme@phme-web.cloudapp.net:50226',
21 ]
22web_servers_pickit = ['phme@pickit-web.cloudapp.net',
23 'phme@pickit-web.cloudapp.net:62152',
24 'phme@pickit-web.cloudapp.net:55165',
25 ]
26
27web_servers_pickit_new = [
28 'phme@webli.northeurope.cloudapp.azure.com:50000',
29 'phme@webli.northeurope.cloudapp.azure.com:50001',
30 'phme@webli.northeurope.cloudapp.azure.com:50002',
31]
32
33dev_web_servers = [
34 'phme@webde.northeurope.cloudapp.azure.com:50000',
35 'phme@webde.northeurope.cloudapp.azure.com:50001'
36]
37dev_worker_servers = ['phme@workerde.northeurope.cloudapp.azure.com:50000']
38worker_servers = ['phme@phme-workers.cloudapp.net']
39
40worker_servers_new = ['phme@workerli.northeurope.cloudapp.azure.com:50000']
41
42dev_search_master_servers = ['phme-dev-data.cloudapp.net']
43search_master_servers = ['phme-search.cloudapp.net', 'phme-search.cloudapp.net:58186']
44
45NUMBER_WORKERS = 3
46NUMBER_WORKERS_DEV = 1
47
48webs_human = {
49 'phme@phme-web.cloudapp.net': 'phme-web-01',
50 'phme@phme-web.cloudapp.net:50226': 'phme-web-02',
51 'phme@pickit-web.cloudapp.net': 'pickit-web-01',
52 'phme@pickit-web.cloudapp.net:62152': 'pickit-web-02',
53 'phme@pickit-web.cloudapp.net:55165': 'pickit-web-03',
54 'phme@webli.northeurope.cloudapp.azure.com:50000': 'pickit-web-01',
55 'phme@webli.northeurope.cloudapp.azure.com:50001': 'pickit-web-02',
56 'phme@webli.northeurope.cloudapp.azure.com:50002': 'pickit-web-03',
57}
58
59webs_human_dev = {
60 'phme@phme-dev.cloudapp.net': 'phme-web',
61}
62
63env.forward_agent = True
64
65
66def publish_slack_message(msg, channel='systemlog'):
67 """
68 Publish message into #systemlog channel at PicHit Slack
69
70 :param msg:
71 :return:
72 """
73 response = requests.post("https://teampickit.slack.com/services/hooks/slackbot?"
74 "token=ZFrKeuxVTXOLuCe8oHDnBZjB&channel=%23{}".format(channel),
75 msg)
76 return response
77
78
79def __deploy_user_dev(username, branch):
80 """
81 Deploys at user development server
82
83 **Attributes**
84
85 * ``username``
86 * ``branch``
87 """
88 # local git push
89 current_branch = local('git rev-parse --symbolic-full-name --abbrev-ref HEAD', capture=True)
90 if current_branch != branch:
91 abort('"deploy_user" command pushes current git branch to your user environment. '
92 'Your current branch is different from branch you want to push. Please do "git checkout {}"'
93 .format(branch))
94 local('git fetch')
95 local('git pull origin {}'.format(branch))
96 local('git push origin {}'.format(branch))
97 git_dir = '/home/{}/pichit.me/phme_faraday'.format(username)
98 with cd(git_dir):
99 run('git fetch')
100 run('git checkout {}'.format(branch))
101 run('git pull origin {}'.format(branch))
102 run('/home/{}/pichit.me/bin/pip install -r configs/dev/dev_requirements.txt'.format(username))
103 run('grunt dist:dev')
104 run('/home/{}/pichit.me/phme_faraday/scripts/make_lowercase.py'.format('phme'))
105 run('/home/{}/pichit.me/bin/python manage.py javascript_settings --static_path {}/static/'
106 .format(username, git_dir))
107 run("sed -i 's/configuration/pichitme_settings/g' {}/static/javascript-settings.js".format(git_dir))
108 run('touch /home/{0}/config/uwsgi.{0}.ini'.format(username))
109
110def __deploy_user_dev_new(username, branch):
111 """
112 Deploys at user development server with new setup for development with load balancer
113
114 We execute this in all web instances for scale set
115
116 **Attributes**
117
118 * ``username``
119 * ``branch``
120 """
121 # local git push
122 current_branch = local('git rev-parse --symbolic-full-name --abbrev-ref HEAD', capture=True)
123 if current_branch != branch:
124 abort('"deploy_user" command pushes current git branch to your user environment. '
125 'Your current branch is different from branch you want to push. Please do "git checkout {}"'
126 .format(branch))
127 local('git fetch')
128 local('git pull origin {}'.format(branch))
129 local('git push origin {}'.format(branch))
130 git_dir = '/home/{}/pichit.me/phme_faraday'.format(username)
131 with cd(git_dir):
132 run('git fetch')
133 run('git checkout {}'.format(branch))
134 run('git pull origin {}'.format(branch))
135 run('/home/{}/pichit.me/bin/pip install -r configs/dev/dev_requirements.txt'.format(username))
136 run('grunt dist:dev')
137 run('/home/{}/pichit.me/phme_faraday/scripts/make_lowercase.py'.format('phme'))
138 run('/home/{}/pichit.me/bin/python manage.py javascript_settings --static_path {}/static/'
139 .format(username, git_dir))
140 run("sed -i 's/configuration/pichitme_settings/g' {}/static/javascript-settings.js".format(git_dir))
141 run('touch /home/{0}/config/uwsgi.{0}.ini'.format(username))
142
143def __deploy_user_dev_new(username, branch):
144 """
145 Deploys at user development server with new setup for development with load balancer
146
147 We execute this in all web instances for scale set
148
149 **Attributes**
150
151 * ``username``
152 * ``branch``
153 """
154 # local git push
155 current_branch = local('git rev-parse --symbolic-full-name --abbrev-ref HEAD', capture=True)
156 if current_branch != branch:
157 abort('"deploy_user" command pushes current git branch to your user environment. '
158 'Your current branch is different from branch you want to push. Please do "git checkout {}"'
159 .format(branch))
160 local('git fetch')
161 local('git pull origin {}'.format(branch))
162 local('git push origin {}'.format(branch))
163 git_dir = '/home/{}/pichit.me/phme_faraday'.format(username)
164 with cd(git_dir):
165 run('git fetch')
166 run('git checkout {}'.format(branch))
167 run('git pull origin {}'.format(branch))
168 run('/home/{}/pichit.me/bin/pip install -r configs/dev/dev_requirements.txt'.format(username))
169 run('grunt dist:dev')
170 run('/home/{}/pichit.me/phme_faraday/scripts/make_lowercase.py'.format('phme'))
171 run('/home/{}/pichit.me/bin/python manage.py javascript_settings --static_path {}/static/'
172 .format(username, git_dir))
173 run("sed -i 's/configuration/pichitme_settings/g' {}/static/javascript-settings.js".format(git_dir))
174 run('touch /home/{0}/config/uwsgi.{0}.ini'.format(username))
175
176
177def __create_deploy_tag():
178 """
179 Create deployment tag
180
181 :return:
182 """
183 deploy_time = datetime.now()
184 # deploy__25_sept_2015
185 deploy_tag = u'deploy__{}'.format(
186 deploy_time.strftime('%d_%b_%Y_%H%M')
187 )
188 # git tag -a deploy__25_sept_2015_1420
189 run('git tag -a {} -m "{}"'.format(
190 deploy_tag,
191 deploy_tag
192 ))
193 run('git push origin {}'.format(
194 deploy_tag
195 ))
196 return deploy_tag
197
198@task
199def merge_master_live():
200 """
201 Merges master branch into live branch
202 """
203 local('git checkout live')
204 local('git fetch')
205 local('git pull origin live')
206 local('git merge origin/master')
207 local('git push origin live')
208
209
210@task
211@hosts(*[web_servers[0]])
212def migrate_database():
213 """
214 Migrate database: new table creation and change table definition: schema changes
215
216 :return:
217 """
218 t_1 = time.time()
219 with cd(git_dir_new):
220 publish_slack_message(u'[{}] Starting migration...'.format(
221 datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
222 run('git fetch')
223 run('git pull origin live')
224 run('/home/phme/pichit.me/bin/python manage.py syncdb')
225 run('/home/phme/pichit.me/bin/python manage.py migrate')
226 publish_slack_message(u'[{}] Migration Done! {} seconds'.format(
227 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
228 int(time.time()-t_1)))
229
230@task
231@hosts(*[dev_web_servers[0]])
232def dev_migrate_database():
233 """
234 Migrate database: new table creation and change table definition: schema changes
235
236 :return:
237 """
238 t_1 = time.time()
239 with cd(git_dir_new):
240 publish_slack_message(u'[{}] Starting migration [dev] ...'.format(
241 datetime.now().strftime("%Y-%m-%d %H:%M:%S")), channel='systemlog-dev')
242 run('git fetch')
243 run('git checkout master')
244 run('git pull origin master')
245 run('/home/phme/pichit.me/bin/pip install -r configs/dev/dev_requirements.txt')
246 run('/home/phme/pichit.me/bin/python manage.py syncdb')
247 run('/home/phme/pichit.me/bin/python manage.py migrate')
248 publish_slack_message(u'[{}] Migration Done! {} seconds [dev]'.format(
249 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
250 int(time.time()-t_1)), channel='systemlog-dev')
251
252@task
253@hosts(*web_servers)
254def deploy_webs(t_start_down=None):
255 """
256 Deploy web servers
257
258 :return:
259 """
260 with cd(git_dir_new):
261 run('git fetch')
262 run('git pull origin live')
263 run("grunt dist:live")
264 run('/home/phme/pichit.me/bin/pip install -r configs/prod/prod_requirements.txt')
265 run('/home/phme/pichit.me/phme_faraday/scripts/make_lowercase.py')
266 run('/home/{}/pichit.me/bin/python manage.py javascript_settings --static_path {}/static/'
267 .format('phme', git_dir_new))
268 run("sed -i 's/configuration/pichitme_settings/g' {}/static/javascript-settings.js".format(git_dir_new))
269 """with cd(dir_task_pane_app):
270 run("grunt dist:live --env=prod")"""
271 if exists('/home/phme/config/uwsgi.pichit.me.ini'):
272 run('touch /home/phme/config/uwsgi.pichit.me.ini')
273 else:
274 run('cp /home/phme/config/uwsgi.pichit.me-src.ini /home/phme/config/uwsgi.pichit.me.ini')
275 publish_slack_message(u'[{}] Deployed web [{}]'.format(
276 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
277 webs_human.get(env.host_string, 'undefined')))
278 if webs_human.get(env.host_string, 'undefined') == 'phme-web-01':
279 if t_start_down and int(time.time()-t_start_down) > 60:
280 downtime = fpformat.fix(float(int(time.time()-t_start_down))/60.0, 1) + ' minutes' if t_start_down \
281 else 'None'
282 else:
283 downtime = str(int(time.time()-t_start_down)) + ' seconds' if t_start_down else 'None'
284 publish_slack_message(u'[{}] Site is up!! downtime: {}'.format(
285 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
286 downtime))
287
288
289@task
290@hosts(*web_servers_pickit)
291def deploy_webs_pickit(t_start_down=None):
292 """
293 Deploy web servers
294
295 :return:
296 """
297 with cd(git_dir_new):
298 run('git fetch')
299 run('git pull origin pickit_live')
300 run("grunt dist:live")
301 run('/home/phme/pichit.me/bin/pip install -r configs/prod/prod_requirements.txt')
302 run('/home/phme/pichit.me/phme_faraday/scripts/make_lowercase.py')
303 run('/home/{}/pichit.me/bin/python manage.py javascript_settings --static_path {}/static/'
304 .format('phme', git_dir_new))
305 run("sed -i 's/configuration/pichitme_settings/g' {}/static/javascript-settings.js".format(git_dir_new))
306 with cd(dir_task_pane_app):
307 run("grunt dist:live --env=prod")
308 if exists('/home/phme/config/uwsgi.pichit.me.ini'):
309 run('touch /home/phme/config/uwsgi.pichit.me.ini')
310 else:
311 run('cp /home/phme/config/uwsgi.pichit.me-src.ini /home/phme/config/uwsgi.pichit.me.ini')
312 publish_slack_message(u'[{}] Deployed web pickit.com [{}]'.format(
313 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
314 webs_human.get(env.host_string, 'undefined')))
315 if webs_human.get(env.host_string, 'undefined') == 'pickit-web-01':
316 if t_start_down and int(time.time()-t_start_down) > 60:
317 downtime = fpformat.fix(float(int(time.time()-t_start_down))/60.0, 1) + ' minutes' if t_start_down \
318 else 'None'
319 else:
320 downtime = str(int(time.time()-t_start_down)) + ' seconds' if t_start_down else 'None'
321 publish_slack_message(u'[{}] Site is up!! downtime: {}'.format(
322 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
323 downtime))
324 # deploy_tag = __create_deploy_tag()
325 # print u'sync_workers_script_zone :: deploy_tag: {}'.format(deploy_tag)
326 # return deploy_tag
327
328
329@task
330@hosts(*dev_web_servers)
331def deploy_dev_webs(t_start_down=None):
332 """
333 Deploy web servers [dev]
334
335 :return:
336 """
337 with cd(git_dir_new):
338 run('git fetch')
339 run('git checkout master')
340 run('git pull origin master')
341 run("grunt dist:dev")
342 run('/home/phme/pichit.me/bin/pip install -r configs/dev/dev_requirements.txt')
343 run('/home/phme/pichit.me/phme_faraday/scripts/make_lowercase.py')
344 run('/home/{}/pichit.me/bin/python manage.py javascript_settings --static_path {}/static/'
345 .format('phme', git_dir_new))
346 run("sed -i 's/configuration/pichitme_settings/g' {}/static/javascript-settings.js".format(git_dir_new))
347 with cd(dir_task_pane_app):
348 run("grunt dist:dev --env=dev")
349 if exists(dir_cmp_app):
350 with cd(dir_cmp_app+'/search-dashboard'):
351 run('git fetch')
352 run('git checkout master')
353 run('git pull origin master')
354 run('bower install')
355 run('grunt build')
356 else:
357 run('mkdir {}'.format(dir_cmp_app))
358 with cd(dir_cmp_app):
359 run('git clone git@bitbucket.org:phme_admin/search-dashboard.git')
360 with cd('./search-dashboard'):
361 run('npm install ')
362 run('sudo npm install bower -g')
363 run('bower install')
364 run('grunt build')
365 if exists('/home/phme/config/uwsgi.pichitmedev.com.ini'):
366 run('touch /home/phme/config/uwsgi.pichitmedev.com.ini')
367 else:
368 run('cp /home/phme/config/uwsgi.pichitmedev.com-src.ini /home/phme/config/uwsgi.pichitmedev.com.ini')
369 publish_slack_message(u'[{}] Deployed web [{}] [dev]'.format(
370 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
371 webs_human_dev.get(env.host_string, 'undefined')), channel='systemlog-dev')
372 if webs_human_dev.get(env.host_string, 'undefined') == 'phme-dev':
373 if t_start_down and int(time.time()-t_start_down) > 60:
374 downtime = fpformat.fix(float(int(time.time()-t_start_down))/60.0, 1) + ' minutes' if t_start_down \
375 else 'None'
376 else:
377 downtime = str(int(time.time()-t_start_down)) + ' seconds' if t_start_down else 'None'
378 publish_slack_message(u'[{}] Site is up!! downtime: {} [dev]'.format(
379 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
380 downtime), channel='systemlog-dev')
381
382@task
383@hosts(*worker_servers)
384def deploy_workers(is_stopped=False):
385 """
386 Deploy workers at phme-worker-01: We restart celery with deployment.
387
388 :return:
389 """
390 env.hosts = worker_servers
391
392 workers = map(lambda x: "worker{}".format(x), range(1, NUMBER_WORKERS+1))
393 for worker in workers:
394 if not is_stopped:
395 sudo('supervisorctl stop phme_celeryd_{}'.format(worker))
396 git_dir_ = '/home/phme/pichit.me/{}/phme_faraday'.format(worker)
397 with cd(git_dir_):
398 run('git fetch')
399 run('git pull origin live')
400 run('/home/phme/pichit.me/bin/pip install -r configs/prod/prod_requirements.txt')
401 sudo('supervisorctl start phme_celeryd_{}'.format(worker))
402 sudo('supervisorctl restart phme_celerybeat')
403 publish_slack_message(u'[{}] I released code on worker [{}]'.format(
404 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
405 worker))
406
407
408@task
409@hosts(*dev_worker_servers)
410def deploy_dev_workers(is_stopped=False):
411 """
412 Deploy workers at phme-dev-worker: We restart celery with deployment.
413
414 :return:
415 """
416 env.hosts = worker_servers
417
418 workers = map(lambda x: "worker{}".format(x), range(1, NUMBER_WORKERS_DEV+1))
419 for worker in workers:
420 if not is_stopped:
421 sudo('supervisorctl stop phme_celeryd_{}'.format(worker))
422 git_dir_ = '/home/phme/pichit.me/{}/phme_faraday'.format(worker)
423 with cd(git_dir_):
424 run('git fetch')
425 run('git checkout master')
426 run('git pull origin master')
427 run('/home/phme/pichit.me/bin/pip install -r configs/dev/dev_requirements.txt')
428 sudo('supervisorctl start phme_celeryd_{}'.format(worker))
429 sudo('supervisorctl restart phme_celerybeat')
430 publish_slack_message(u'[{}] I released code on worker [{}] [dev]'.format(
431 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
432 worker), channel='systemlog-dev')
433
434@task
435@hosts(*[worker_servers[0]])
436def sync_workers_script_zone():
437 git_dir_ = '/home/phme/pichit.me/phme_faraday'
438 with cd(git_dir_):
439 run('git fetch')
440 run('git pull origin live')
441 run('/home/phme/pichit.me/bin/pip install -r configs/prod/prod_requirements.txt')
442 deploy_tag = __create_deploy_tag()
443 print u'sync_workers_script_zone :: deploy_tag: {}'.format(deploy_tag)
444 return deploy_tag
445
446@task
447@hosts(*[dev_worker_servers[0]])
448def sync_dev_worker():
449 git_dir_ = '/home/phme/pichit.me/phme_faraday'
450 with cd(git_dir_):
451 run('git fetch')
452 run('git checkout master')
453 run('git pull origin master')
454 run('/home/phme/pichit.me/bin/pip install -r configs/dev/dev_requirements.txt')
455
456@task
457@hosts(*web_servers)
458def stop_webs():
459 """
460 Stop all web servers for PicHit
461
462 :return:
463 """
464 env.hosts = web_servers
465 web_config_dir = '/home/phme/config'
466 with cd(web_config_dir):
467 if exists(web_config_dir + '/uwsgi.pichit.me.ini'):
468 run('rm uwsgi.pichit.me.ini')
469 publish_slack_message(u'[{}] I put down web [{}]'.format(
470 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
471 webs_human.get(env.host_string, 'undefined')))
472
473@task
474@hosts(*web_servers_pickit)
475def stop_pickit_webs():
476 """
477 Stop all web servers for PicHit
478
479 :return:
480 """
481 env.hosts = web_servers
482 web_config_dir = '/home/phme/config'
483 with cd(web_config_dir):
484 if exists(web_config_dir + '/uwsgi.pichit.me.ini'):
485 run('rm uwsgi.pichit.me.ini')
486 publish_slack_message(u'[{}] I put down web [{}]'.format(
487 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
488 webs_human.get(env.host_string, 'undefined')))
489
490@task
491@hosts(*dev_web_servers)
492def stop_dev_webs():
493 """
494 Stop all [dev] web servers for PicHit
495
496 :return:
497 """
498 env.hosts = dev_web_servers
499 web_config_dir = '/home/phme/config'
500 with cd(web_config_dir):
501 if exists(web_config_dir + '/uwsgi.pichitmedev.com.ini'):
502 run('rm uwsgi.pichitmedev.com.ini')
503 publish_slack_message(u'[{}] I put down web [{}]'.format(
504 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
505 webs_human_dev.get(env.host_string, 'undefined')), channel='systemlog-dev')
506
507@task
508@hosts(*web_servers)
509def restart_webs(deployment_name=''):
510 """
511 Restart all web servers for PicHit
512
513 :return:
514 """
515 env.hosts = web_servers
516 web_config_dir = '/home/phme/config'
517 if deployment_name:
518 deployment_name = '"' + deployment_name + '" '
519 if webs_human.get(env.host_string, 'undefined') == 'phme-web-01':
520 publish_slack_message(u'[{}] Restart webs {} ...'.format(
521 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
522 deployment_name))
523 with cd(web_config_dir):
524 if exists('/home/phme/config/uwsgi.pichit.me.ini'):
525 run('touch /home/phme/config/uwsgi.pichit.me.ini')
526 else:
527 run('cp /home/phme/config/uwsgi.pichit.me-src.ini /home/phme/config/uwsgi.pichit.me.ini')
528 publish_slack_message(u'[{}] I restart [{}]'.format(
529 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
530 webs_human.get(env.host_string, 'undefined')))
531
532
533@task
534@hosts(*web_servers)
535def start_webs():
536 """
537 Start all web servers for PicHit
538
539 :return:
540 """
541 if exists('/home/phme/config/uwsgi.pichit.me.ini'):
542 run('touch /home/phme/config/uwsgi.pichit.me.ini')
543 else:
544 run('cp /home/phme/config/uwsgi.pichit.me-src.ini /home/phme/config/uwsgi.pichit.me.ini')
545 publish_slack_message(u'[{}] I put up web [{}]'.format(
546 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
547 webs_human.get(env.host_string, 'undefined')))
548
549
550@task
551@hosts(*worker_servers)
552def stop_workers():
553 """
554 Stop workers
555
556 :return:
557 """
558 env.hosts = worker_servers
559 workers = map(lambda x: "worker{}".format(x), range(1, NUMBER_WORKERS+1))
560 for worker in workers:
561 sudo('supervisorctl stop phme_celeryd_{}'.format(worker))
562 publish_slack_message(u'[{}] I put down worker [{}]'.format(
563 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
564 worker))
565
566
567@task
568@hosts(*dev_worker_servers)
569def stop_dev_workers():
570 """
571 Stop [dev] workers
572
573 :return:
574 """
575 env.hosts = dev_worker_servers
576 workers = map(lambda x: "worker{}".format(x), range(1, NUMBER_WORKERS_DEV+1))
577 for worker in workers:
578 sudo('supervisorctl stop phme_celeryd_{}'.format(worker))
579 publish_slack_message(u'[{}] I put down worker [{}] [dev]'.format(
580 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
581 worker), channel='systemlog-dev')
582
583@task
584@hosts(*worker_servers)
585def start_workers():
586 """
587 Start workers
588
589 :return:
590 """
591 env.hosts = worker_servers
592 workers = map(lambda x: "worker{}".format(x), range(1, NUMBER_WORKERS+1))
593 for worker in workers:
594 sudo('supervisorctl start phme_celeryd_{}'.format(worker))
595 publish_slack_message(u'[{}] I put up worker [{}]'.format(
596 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
597 worker))
598
599
600@task
601@runs_once
602def deploy_live(deployment_name=''):
603 """
604 Deploy live for new releases with schema migrations
605
606 :return:
607 """
608 if deployment_name:
609 deployment_name = '"' + deployment_name + '" '
610 publish_slack_message(u'[{}] Start deployment {} ...'.format(
611 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
612 deployment_name))
613
614 t_1 = time.time()
615 t_start_down = time.time()
616 publish_slack_message(u'[{}] Stopping workers...'.format(
617 datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
618 execute(stop_workers, hosts=worker_servers)
619 publish_slack_message(u'[{}] All workers stopped! {} seconds'.format(
620 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
621 int(time.time()-t_1)))
622
623 t_1 = time.time()
624 publish_slack_message(u'[{}] Stopping webs...'.format(
625 datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
626 execute(stop_webs, hosts=web_servers)
627 execute(stop_pickit_webs, hosts=web_servers_pickit)
628 publish_slack_message(u'[{}] All webs stopped! {} seconds'.format(
629 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
630 int(time.time()-t_1)))
631
632 execute(migrate_database, hosts=[web_servers[0]])
633
634 t_1 = time.time()
635 publish_slack_message(u'[{}] Deploying webs...'.format(
636 datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
637 execute(deploy_webs, hosts=web_servers, t_start_down=t_start_down)
638 publish_slack_message(u'[{}] All webs deployed! {} seconds'.format(
639 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
640 int(time.time()-t_1)))
641
642 t_1 = time.time()
643 worker_data = execute(sync_workers_script_zone)
644 publish_slack_message(u'[{}] Deploying workers...'.format(
645 datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
646 execute(deploy_workers, hosts=worker_servers, is_stopped=True)
647 publish_slack_message(u'[{}] All workers deployed! {} seconds'.format(
648 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
649 int(time.time()-t_1)))
650
651 deploy_tag = worker_data[worker_servers[0]]
652
653 publish_slack_message(u'[{}] Finished deployment!!! {} minutes. deploy tag: {}'.format(
654 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
655 fpformat.fix(float(int(time.time()-t_start_down))/60.0, 1),
656 deploy_tag))
657
658
659@task
660@runs_once
661def deploy_live_no_db(deployment_name=''):
662 """
663 Deploy live without schema migrations
664
665 :return:
666 """
667 t_start_down = time.time()
668 if deployment_name:
669 deployment_name = '"' + deployment_name + '" '
670 publish_slack_message(u'[{}] Start deployment no migrations {} ...'.format(
671 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
672 deployment_name))
673
674 t_1 = time.time()
675 publish_slack_message(u'[{}] Deploying webs...'.format(
676 datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
677 execute(deploy_webs, hosts=web_servers)
678 publish_slack_message(u'[{}] All webs deployed! {} seconds'.format(
679 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
680 int(time.time()-t_1)))
681
682 t_1 = time.time()
683 publish_slack_message(u'[{}] Deploying workers...'.format(
684 datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
685 worker_data = execute(sync_workers_script_zone)
686 execute(deploy_workers, hosts=worker_servers,)
687 publish_slack_message(u'[{}] All workers deployed! {} seconds'.format(
688 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
689 int(time.time()-t_1)))
690
691 deploy_tag = worker_data[worker_servers[0]]
692
693 publish_slack_message(u'[{}] Deployment done! {} minutes. deploy tag: {}'.format(
694 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
695 fpformat.fix(float(int(time.time()-t_start_down))/60.0, 1),
696 deploy_tag,
697 )
698 )
699
700
701@task
702@runs_once
703def deploy_live_workers_no_db(deployment_name=''):
704 """
705 Deploy live to workers without schema migrations
706
707 :return:
708 """
709 t_1 = time.time()
710 if deployment_name:
711 deployment_name = '"' + deployment_name + '" '
712 publish_slack_message(u'[{}] Start worker deployment {} ...'.format(
713 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
714 deployment_name))
715 worker_data = execute(sync_workers_script_zone)
716 execute(deploy_workers, hosts=worker_servers)
717
718 deploy_tag = worker_data[worker_servers[0]]
719
720 publish_slack_message(u'[{}] Worker deployment done! {} minutes. deploy tag: {}'.format(
721 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
722 fpformat.fix(float(int(time.time()-t_1))/60.0, 1),
723 deploy_tag
724 )
725 )
726
727@task
728@runs_once
729def deploy_live_web_no_db(deployment_name=''):
730 """
731 Deploy live to webs with no data migrations
732
733 :return:
734 """
735 t_1 = time.time()
736 if deployment_name:
737 deployment_name = '"' + deployment_name + '" '
738 publish_slack_message(u'[{}] Start web deployment {} ...'.format(
739 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
740 deployment_name))
741 execute(deploy_webs, hosts=web_servers, t_start_down=t_1)
742 worker_data = execute(sync_workers_script_zone)
743 deploy_tag = worker_data[worker_servers[0]]
744
745 publish_slack_message(u'[{}] Web deployment done! {} minutes. deploy tag: {}'.format(
746 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
747 fpformat.fix(float(int(time.time()-t_1))/60.0, 1),
748 deploy_tag
749 )
750 )
751
752
753@task
754@runs_once
755def deploy_live_pickit_webs(deployment_name=''):
756 """
757 Deploy only to pickit.com webs
758
759 :return:
760 """
761 t_1 = time.time()
762 if deployment_name:
763 deployment_name = '"' + deployment_name + '" '
764 publish_slack_message(u'[{}] Start web deployment pickit.com {} ...'.format(
765 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
766 deployment_name))
767 execute(deploy_webs_pickit, hosts=web_servers_pickit, t_start_down=t_1)
768 # deploy_tag = execute(deploy_webs_pickit, hosts=web_servers_pickit, t_start_down=t_1)
769
770 publish_slack_message(u'[{}] Web deployment pickit.com done! {} minutes.'.format(
771 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
772 fpformat.fix(float(int(time.time()-t_1))/60.0, 1)
773 )
774 )
775
776
777@task
778@runs_once
779def deploy_live_pickit_webs_new(deployment_name=''):
780 """
781 Deploy only to pickit.com webs new setup
782
783 :return:
784 """
785 t_1 = time.time()
786 if deployment_name:
787 deployment_name = '"' + deployment_name + '" '
788 publish_slack_message(u'[{}] Start web deployment pickit.com [NEW SETUP] {} ...'.format(
789 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
790 deployment_name))
791 execute(deploy_webs_pickit, hosts=web_servers_pickit_new, t_start_down=t_1)
792 # deploy_tag = execute(deploy_webs_pickit, hosts=web_servers_pickit, t_start_down=t_1)
793
794 publish_slack_message(u'[{}] Web deployment pickit.com done! {} minutes. [NEW SETUP]'.format(
795 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
796 fpformat.fix(float(int(time.time()-t_1))/60.0, 1)
797 )
798 )
799
800
801@task
802@hosts(*worker_servers[0:1])
803def deploy_cron_scripts():
804 """
805 Deploys CRON script directly to first node of worker nodes (CRON node) without
806 restarting celery through supervisor
807
808 :return:
809 """
810 with cd(git_dir_new):
811 run('git fetch')
812 run('git pull origin live')
813 run('/home/phme/pichit.me/bin/pip install -r configs/prod/prod_requirements.txt')
814
815
816@task
817@hosts('phme@phme-dev.cloudapp.net',)
818def create_schemas_dev():
819 """
820 Create schema migrations at faraday and push to repository
821 :return:
822 """
823 env.warn_only = True
824 git_dir = '/home/phme/pichit.me/phme_django'
825 with cd(git_dir):
826 run('git fetch')
827 run('git pull origin master')
828 run('/home/phme/pichit.me/bin/python manage.py schemamigration account --auto')
829 run('/home/phme/pichit.me/bin/python manage.py schemamigration comments --auto')
830 run('/home/phme/pichit.me/bin/python manage.py schemamigration favourites --auto')
831 run('/home/phme/pichit.me/bin/python manage.py schemamigration main --auto')
832 run('/home/phme/pichit.me/bin/python manage.py schemamigration newsletter --auto')
833 run('/home/phme/pichit.me/bin/python manage.py schemamigration notifications --auto')
834 run('/home/phme/pichit.me/bin/python manage.py schemamigration products --auto')
835 run('/home/phme/pichit.me/bin/python manage.py schemamigration promotions --auto')
836 run('/home/phme/pichit.me/bin/python manage.py schemamigration rawpages --auto')
837 run('/home/phme/pichit.me/bin/python manage.py schemamigration rewards --auto')
838 run('/home/phme/pichit.me/bin/python manage.py schemamigration simplecurrencies --auto')
839 run('/home/phme/pichit.me/bin/python manage.py schemamigration tags --auto')
840 run('git add apps/account/migrations/*.py apps/comments/migrations/*.py apps/favourites/migrations/*.py '
841 'apps/main/migrations/*.py apps/newsletter/migrations/*.py apps/notifications/migrations/*.py '
842 'apps/products/migrations/*.py apps/promotions/migrations/*.py apps/rawpages/migrations/*.py '
843 'apps/rewards/migrations/*.py apps/simplecurrencies/migrations/*.py apps/tags/migrations/*.py')
844 run("git commit -a -m 'Schema migrations'")
845 run('git push origin master')
846
847
848@task
849@hosts('phme@phme-dev.cloudapp.net',)
850def deploy_dev_web(deployment_name=''):
851 """
852 Deploys at development server: web and worker
853 """
854 git_dir = '/home/phme/pichit.me/phme_faraday'
855
856 if deployment_name:
857 deployment_name = '"' + deployment_name + '" '
858 publish_slack_message(u'[{}] Start development deployment {} ...'.format(
859 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
860 deployment_name))
861
862 with cd(git_dir):
863 run('git fetch')
864 run('git checkout master')
865 run('git pull origin master')
866 run("grunt dist:dev")
867 run('/home/phme/pichit.me/bin/pip install -r configs/dev/dev_requirements.txt')
868 run('/home/phme/pichit.me/phme_faraday/scripts/make_lowercase.py')
869 run('/home/{}/pichit.me/bin/python manage.py javascript_settings --static_path {}/static/'
870 .format('phme', git_dir_new))
871 run("sed -i 's/configuration/pichitme_settings/g' {}/static/javascript-settings.js".format(git_dir_new))
872 if exists('/home/phme/config/uwsgi.pichitmedev.com.ini'):
873 run('touch /home/phme/config/uwsgi.pichitmedev.com.ini')
874 else:
875 run('cp /home/phme/config/uwsgi.pichitmedev.com-src.ini.ini /home/phme/config/uwsgi.pichitmedev.com.ini.ini')
876 publish_slack_message(u'[{}] Deployed web development [{}]'.format(
877 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
878 'phme-dev'))
879
880 execute(sync_dev_worker)
881
882
883@task
884@runs_once
885def deploy_dev(deployment_name=''):
886
887 if deployment_name:
888 deployment_name = '"' + deployment_name + '" '
889 publish_slack_message(u'[{}] Start [dev] deployment {} ...'.format(
890 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
891 deployment_name), channel='systemlog-dev')
892
893 t_1 = time.time()
894 t_start_down = time.time()
895 publish_slack_message(u'[{}] Stopping [dev] workers...'.format(
896 datetime.now().strftime("%Y-%m-%d %H:%M:%S")), channel='systemlog-dev')
897 execute(stop_dev_workers, hosts=dev_worker_servers)
898 publish_slack_message(u'[{}] All [dev] workers stopped! {} seconds'.format(
899 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
900 int(time.time()-t_1)), channel='systemlog-dev')
901
902 t_1 = time.time()
903 publish_slack_message(u'[{}] Stopping [dev] webs...'.format(
904 datetime.now().strftime("%Y-%m-%d %H:%M:%S")), channel='systemlog-dev')
905 execute(stop_dev_webs, hosts=dev_web_servers)
906 publish_slack_message(u'[{}] All [dev] webs stopped! {} seconds'.format(
907 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
908 int(time.time()-t_1)), channel='systemlog-dev')
909
910 execute(dev_migrate_database, hosts=[dev_web_servers[0]])
911
912 t_1 = time.time()
913 publish_slack_message(u'[{}] Deploying [dev] webs...'.format(
914 datetime.now().strftime("%Y-%m-%d %H:%M:%S")), channel='systemlog-dev')
915 execute(deploy_dev_webs, hosts=dev_web_servers, t_start_down=t_start_down)
916 publish_slack_message(u'[{}] All [dev]webs deployed! {} seconds'.format(
917 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
918 int(time.time()-t_1)), channel='systemlog-dev')
919
920 t_1 = time.time()
921 execute(sync_dev_worker)
922 publish_slack_message(u'[{}] Deploying [dev] workers...'.format(
923 datetime.now().strftime("%Y-%m-%d %H:%M:%S")), channel='systemlog-dev')
924 execute(deploy_dev_workers, hosts=dev_worker_servers, is_stopped=True)
925 publish_slack_message(u'[{}] All [dev] workers deployed! {} seconds'.format(
926 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
927 int(time.time()-t_1)), channel='systemlog-dev')
928
929 publish_slack_message(u'[{}] Finished [dev] deployment!!! {} minutes'.format(
930 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
931 fpformat.fix(float(int(time.time()-t_start_down))/60.0, 1)), channel='systemlog-dev')
932
933
934@task
935@hosts('phme@phme-dev.cloudapp.net',)
936def dev_grunt_env():
937 """
938 Create grunt node environment at Faraday
939 :return:
940 """
941 git_dir = '/home/phme/pichit.me/phme_django'
942 with cd(git_dir):
943 run('cp configs/dev/phme-node/package.json .')
944 run('npm install')
945 run('rm -f package.json')
946
947
948@task
949@hosts(*dev_web_servers)
950def create_user_env(user, git_user):
951 """
952 Create user environment
953
954 :param user:
955 :param git_user:
956 :return:
957 """
958 # all ops changing from phme user to username once has been created
959 # 1. create user, add sudo group to it
960 home = '/home/{}'.format(user)
961 group = 'phme'
962 sudo('useradd {0} -m -s /bin/bash'.format(user))
963 sudo('adduser {} sudo'.format(user))
964 sudo('sudo passwd {}'.format(user))
965 # 2. create .ssh and authorized_keys with phmefaraday.key
966 sudo('mkdir {}/.ssh'.format(home))
967 sudo('chown {1}.{2} {0}/.ssh'.format(home, user, group))
968 sudo('cp /home/phme/user_envs/.ssh/authorized_keys {}/.ssh'.format(home))
969 sudo('chown {1}.{2} {0}/.ssh/authorized_keys'.format(home, user, group))
970 # Append profile settings:
971 sudo("sed -i '$ a\export DJANGO_SETTINGS_MODULE=\"settings.development\"' /home/{}/.profile".format(user))
972 sudo("sed -i '$ a\export CLARIFAI_APP_ID=c49aGFGUQxeb7Nc8-YTCMQIu_RPrFxbQKBPD4NzYexport "
973 "CLARIFAI_APP_ID=c49aGFGUQxeb7Nc8-YTCMQIu_RPrFxbQKBPD4NzY' /home/{}/.profile".format(user))
974 sudo("sed -i '$ a\export CLARIFAI_APP_SECRET=cGmH3_TdBRbDrQcGUH9Kcj_Qju-poqjn_NYThp8I' "
975 "/home/{}/.profile".format(user))
976 # create log directory
977 sudo('mkdir {}/log'.format(home))
978 sudo('cp /home/phme/user_envs/base.log {0}/log/django.log'.format(home, user))
979 sudo('cp /home/phme/user_envs/base.log {0}/log/uwsgi.log'.format(home, user))
980 sudo('cp /home/phme/user_envs/base.log {0}/log/phme_crowdflower.log'.format(home, user))
981 sudo('cp /home/phme/user_envs/base.log {0}/log/phme_search.log'.format(home, user))
982 sudo('chown {1}.{2} -R {0}/log'.format(home, user, group))
983 sudo('chmod 664 {0}/log/*'.format(home))
984 # create scripts directory
985 sudo('mkdir {}/scripts'.format(home))
986 sudo('chown {1}.{2} {0}/scripts'.format(home, user, group))
987 sudo('mkdir {}/config'.format(home))
988 sudo('chown {1}.{2} {0}/config'.format(home, user, group))
989 # nginx.$user uwsgi.$user.ini uwsgi.$user-src.ini
990 sudo('cp /home/phme/user_envs/uwsgi.ini {0}/config/uwsgi.{1}-src.ini'.format(home, user))
991 sudo('chown {1}.{2} {0}/config/uwsgi.{1}-src.ini'.format(home, user, group))
992 sudo('cp /home/phme/user_envs/uwsgi.ini {0}/config/uwsgi.{1}.ini'.format(home, user))
993 sudo('chown {1}.{2} {0}/config/uwsgi.{1}.ini'.format(home, user, group))
994 # create virtualenv pichit.me
995 with cd('{}/'.format(home)):
996 sudo('virtualenv pichit.me')
997 sudo('chown {0}.{2} -R {1}/pichit.me'.format(user, home, group))
998 # clone repository
999 with cd('{}/pichit.me/'.format(home)):
1000 sudo('git clone https://{}@bitbucket.org/clasperson/phme_faraday.git'.format(git_user))
1001 sudo('chown {0}.{1} -R phme_faraday'.format(user, group))
1002 # install requirements
1003 sudo('{0}/pichit.me/bin/pip install -r {0}/pichit.me/phme_faraday/configs/dev/dev_requirements.txt'.format(home))
1004 sudo('chown {0}.{2} -R {1}/pichit.me'.format(user, home, group))
1005 # copy azure storage backend
1006 sudo('cp /home/phme/pichit.me/lib/python2.7/site-packages/storages/backends/azure_storage.py '
1007 '{}/pichit.me/lib/python2.7/site-packages/storages/backends/'.format(home))
1008 # create nginx file
1009 sudo('cp /home/phme/user_envs/nginx.config {0}/config/nginx.{1}'.format(home, user))
1010 sudo('chown {1}.{2} {0}/config/nginx.{1}'.format(home, user, group))
1011 sudo('python /home/phme/user_envs/scripts/subs_nginx.py {}'.format(user))
1012 # copy _settings_local.py with standard main data
1013 sudo('cp /home/phme/user_envs/_settings_local.py {}/pichit.me/phme_faraday/settings/development/settings_local.py'
1014 .format(home))
1015 sudo('chown {0}.{2} {1}/pichit.me/phme_faraday/settings/development/settings_local.py'.format(
1016 user, home, group))
1017 sudo('python /home/phme/user_envs/scripts/subs_settings.py {}'.format(user))
1018 with cd('/etc/nginx/sites-enabled'):
1019 sudo('ln -s {0}/config/nginx.{1}'.format(home, user))
1020 # install node and basic deployment
1021 with settings(password=getpass.getpass('enter password for %s: ' % user),
1022 sudo_prefix="su %s -c " % user,
1023 sudo_prompt="Password:"):
1024 with cd('{}/pichit.me/phme_faraday'.format(home)):
1025 sudo('cp {0}/pichit.me/phme_faraday/configs/dev/phme-node/package.json .'.format(home), user=user)
1026 sudo('npm install', user=user)
1027 sudo('rm -f package.json', user=user)
1028 sudo('grunt dist:dev', user=user)
1029 with cd('/etc/uwsgi-emperor/vassals/'):
1030 sudo('ln -s {0}/config/uwsgi.{1}.ini'.format(home, user))
1031 # restart nginx
1032 sudo('service nginx restart')
1033
1034
1035@task
1036@hosts('jalegre@phme-dev.cloudapp.net',)
1037def deploy_jalegre(branch):
1038 """
1039 Deploy jalegre server
1040 :param branch:
1041 :return:
1042 """
1043 __deploy_user_dev('jalegre', branch)
1044
1045
1046@task
1047def deploy_user(branch):
1048 """
1049 Deploy user server
1050 :param branch:
1051 :return:
1052 """
1053 __deploy_user_dev(env.user, branch)
1054
1055@task
1056def deploy_user_new(branch):
1057 """
1058 Deploy user server
1059 :param branch:
1060 :return:
1061 """
1062 __deploy_user_dev_new(env.user, branch)
1063
1064@task
1065def deploy_user_new(branch):
1066 """
1067 Deploy user server
1068 :param branch:
1069 :return:
1070 """
1071 __deploy_user_dev_new(env.user, branch)
1072
1073
1074@task
1075@hosts('jalegre@phme-dev.cloudapp.net',)
1076def grunt_env_jalegre():
1077 """
1078 Create grunt/node environment for jalegre
1079 :return:
1080 """
1081 git_dir = '/home/jalegre/pichit.me/phme_faraday'
1082 with cd(git_dir):
1083 run('cp configs/dev/phme-node/package.json .')
1084 run('npm install')
1085 run('rm -f package.json')
1086
1087
1088@task
1089@hosts('zilnys@phme-dev.cloudapp.net',)
1090def deploy_zilnys(branch):
1091 """
1092 Deploy zilnys server
1093 :param branch:
1094 :return:
1095 """
1096 __deploy_user_dev('zilnys', branch)
1097
1098
1099@task
1100@hosts('zilnys@phme-dev.cloudapp.net',)
1101def grunt_env_zilnys():
1102 """
1103 Create grunt/node environment for zilnys
1104 :return:
1105 """
1106 git_dir = '/home/zilnys/pichit.me/phme_faraday'
1107 with cd(git_dir):
1108 run('cp configs/dev/phme-node/package.json .')
1109 run('npm install')
1110 run('rm -f package.json')
1111
1112
1113@task
1114@hosts('clasperson@phme-dev.cloudapp.net',)
1115def deploy_clasperson(branch):
1116 """
1117 Deploy clasperson server
1118 :param branch:
1119 :return:
1120 """
1121 __deploy_user_dev('clasperson', branch)
1122
1123
1124@task
1125@hosts('clasperson@phme-dev.cloudapp.net',)
1126def grunt_env_clasperson():
1127 """
1128 Create grunt/node environment for clasperson
1129 :return:
1130 """
1131 git_dir = '/home/clasperson/pichit.me/phme_faraday'
1132 with cd(git_dir):
1133 run('cp configs/dev/phme-node/package.json .')
1134 run('npm install')
1135 run('rm -f package.json')
1136
1137
1138@task
1139@hosts(*dev_web_servers)
1140def deploy_dev_fast(deployment_name=''):
1141 """
1142 Deploys at development server without migrations and requirements and celeryd restart
1143 """
1144 git_dir = '/home/phme/pichit.me/phme_faraday'
1145
1146 if deployment_name:
1147 deployment_name = '"' + deployment_name + '" '
1148 publish_slack_message(u'[{}] Start development web deployment {} ...'.format(
1149 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
1150 deployment_name), channel='systemlog-dev')
1151
1152 with cd(git_dir):
1153 run('git fetch')
1154 run('git checkout master')
1155 run('git pull origin master')
1156 run("grunt dist:dev")
1157 run('/home/phme/pichit.me/bin/pip install -r configs/dev/dev_requirements.txt')
1158 run('/home/phme/pichit.me/phme_faraday/scripts/make_lowercase.py')
1159 run('/home/{}/pichit.me/bin/python manage.py javascript_settings --static_path {}/static/'
1160 .format('phme', git_dir_new))
1161 run("sed -i 's/configuration/pichitme_settings/g' {}/static/javascript-settings.js".format(git_dir_new))
1162 with cd(dir_task_pane_app):
1163 run("grunt dist:dev --env=dev")
1164 if exists('/home/phme/config/uwsgi.pichitmedev.com.ini'):
1165 run('touch /home/phme/config/uwsgi.pichitmedev.com.ini')
1166 else:
1167 run('cp /home/phme/config/uwsgi.pichitmedev.com-src.ini.ini /home/phme/config/uwsgi.pichitmedev.com.ini.ini')
1168 execute(sync_dev_worker)
1169 publish_slack_message(u'[{}] Deployed development web [{}]'.format(
1170 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
1171 'phme-dev'), channel='systemlog-dev')
1172
1173
1174@task
1175@hosts('pickit@phme-dev.cloudapp.net',)
1176def deploy_dev_fast_pickit(deployment_name=''):
1177 """
1178 Deploys at development server without migrations and requirements and celeryd restart
1179 """
1180 git_dir = '/home/pickit/pichit.me/phme_faraday'
1181
1182 if deployment_name:
1183 deployment_name = '"' + deployment_name + '" '
1184 publish_slack_message(u'[{}] Start development [PickIt] web deployment {} ...'.format(
1185 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
1186 deployment_name), channel='systemlog-dev')
1187
1188 with cd(git_dir):
1189 run('git fetch')
1190 run('git checkout pickit_master')
1191 run('git pull origin pickit_master')
1192 run("grunt dist:dev")
1193 run('/home/pickit/pichit.me/bin/pip install -r configs/dev/dev_requirements.txt')
1194 run('/home/pickit/pichit.me/phme_faraday/scripts/make_lowercase.py')
1195 run('/home/{}/pichit.me/bin/python manage.py javascript_settings --static_path {}/static/'
1196 .format('pickit', git_dir_new_pickit))
1197 run("sed -i 's/configuration/pichitme_settings/g' {}/static/javascript-settings.js".format(
1198 git_dir_new_pickit))
1199 if exists('/home/pickit/config/uwsgi.pickit.ini'):
1200 run('touch /home/pickit/config/uwsgi.pickit.ini')
1201 else:
1202 run('cp /home/pickit/config/uwsgi.pickit-src.ini /home/pickit/config/uwsgi.pickit.ini')
1203 # execute(sync_dev_worker)
1204 publish_slack_message(u'[{}] Deployed development [PickIt] web [{}]'.format(
1205 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
1206 'phme-dev'), channel='systemlog-dev')
1207
1208
1209@task
1210@hosts('phme@phme-dev.cloudapp.net',)
1211def export_search_mappings():
1212 """
1213 Generate mappings file at development server
1214 """
1215 with cd('/home/phme/pichit.me/phme_django'):
1216 run('/home/phme/pichit.me/bin/python manage.py export_search_mappings '
1217 '--path /home/phme/search/search_mappings.json -v 1')
1218 with cd('/home/phme'):
1219 run('scp -i .ssh/phmefaraday.key search/search_mappings.json phme@phme-live.cloudapp.net:/home/phme/search')
1220
1221
1222@task
1223@hosts(*(web_servers+dev_worker_servers+worker_servers))
1224def install_enchant():
1225 run('sudo apt-get install python-enchant')
1226
1227
1228@task
1229@hosts(*dev_search_master_servers)
1230def install_dev_synonyms():
1231 with cd('/home/phme/downloads'):
1232 run('wget http://wordnetcode.princeton.edu/3.0/WNprolog-3.0.tar.gz')
1233 run('tar -xzvf WNprolog-3.0.tar.gz')
1234 with cd('/etc/elasticsearch'):
1235 try:
1236 sudo('mkdir analysis')
1237 except:
1238 pass
1239 sudo('cp /home/phme/downloads/prolog/wn_s.pl /etc/elasticsearch/analysis/wn_s.pl')
1240
1241
1242@task
1243@hosts(*web_servers)
1244def create_cms_live_environment_pichit_me():
1245 """
1246 Create environment for django_cms
1247
1248 :return:
1249 """
1250 # Create envs dir in case does not exist
1251 if not exists('/home/phme/envs'):
1252 with cd('/home/phme'):
1253 run('mkdir envs')
1254 # Create env
1255 if not exists('/home/phme/envs/phme_cms_env'):
1256 with cd('/home/phme/envs'):
1257 run('virtualenv phme_cms_env')
1258 # Create directory phme_cms and have phme_cms repository
1259 with cd('/home/phme'):
1260 if not exists('/home/phme/phme_cms'):
1261 run('git clone https://jalegre39@bitbucket.org/phme_admin/phme_cms.git')
1262 # Install requirements into environment
1263 run('/home/phme/envs/phme_cms_env/bin/pip install -r /home/phme/phme_cms/requirements.txt')
1264 # Copy uwsgi.cms.pichitmedev.com.ini
1265 run('cp /home/phme/pichit.me/phme_faraday/configs/prod/cms/pichit.me/uwsgi.cms.ini '
1266 '/home/phme/config/uwsgi.cms.ini')
1267 run('cp /home/phme/pichit.me/phme_faraday/configs/prod/cms/pichit.me/uwsgi.cms.ini '
1268 '/home/phme/config/uwsgi.cms-src.ini')
1269 # Copy nginx.cms.pichitmedev.com
1270 run('cp /home/phme/pichit.me/phme_faraday/configs/prod/cms/pichit.me/nginx.cms '
1271 '/home/phme/config/nginx.cms')
1272 # Link vassal in emperor
1273 if not exists('/etc/uwsgi-emperor/vassals/uwsgi.cms.ini'):
1274 with cd('/etc/uwsgi-emperor/vassals'):
1275 sudo('ln -s /home/phme/config/uwsgi.cms.ini')
1276 # Link nginx
1277 if not exists('/etc/nginx/sites-enabled/nginx.cms'):
1278 with cd('/etc/nginx/sites-enabled'):
1279 sudo('ln -s /home/phme/config/nginx.cms')
1280 run('touch /home/phme/config/uwsgi.cms.ini')
1281 sudo('service nginx restart')
1282 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py syncdb '
1283 '--settings cms_phme.settings.live')
1284 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py migrate '
1285 '--settings cms_phme.settings.live')
1286 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py collectstatic '
1287 '--settings cms_phme.settings.live')
1288 run('cp /home/phme/pichit.me/phme_faraday/configs/prod/cms/pichit.me/nginx.pichit.me '
1289 '/home/phme/config/nginx.pichit.me')
1290 sudo('service nginx restart')
1291
1292
1293@task
1294@hosts(*[web_servers[0]])
1295def create_cms_live_environment_load_data():
1296 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py loaddata '
1297 '/home/phme/phme_cms/fixtures/for_first_live_deployment.json '
1298 '--settings cms_phme.settings.live')
1299
1300
1301@task
1302@hosts(*web_servers_pickit)
1303def create_cms_live_environment_pickit():
1304 """
1305 Create environment for django_cms
1306
1307 :return:
1308 """
1309 # Create envs dir in case does not exist
1310 if not exists('/home/phme/envs'):
1311 with cd('/home/phme'):
1312 run('mkdir envs')
1313 # Create env
1314 if not exists('/home/phme/envs/phme_cms_env'):
1315 with cd('/home/phme/envs'):
1316 run('virtualenv phme_cms_env')
1317 # Create directory phme_cms and have phme_cms repository
1318 with cd('/home/phme'):
1319 if not exists('/home/phme/phme_cms'):
1320 run('git clone https://jalegre39@bitbucket.org/phme_admin/phme_cms.git')
1321 # Install requirements into environment
1322 run('/home/phme/envs/phme_cms_env/bin/pip install -r /home/phme/phme_cms/requirements.txt')
1323 # Copy uwsgi.cms.pichitmedev.com.ini
1324 run('cp /home/phme/pichit.me/phme_faraday/configs/prod/cms/pickit/uwsgi.cms.ini '
1325 '/home/phme/config/uwsgi.cms.ini')
1326 run('cp /home/phme/pichit.me/phme_faraday/configs/prod/cms/pickit/uwsgi.cms.ini '
1327 '/home/phme/config/uwsgi.cms-src.ini')
1328 # Copy nginx.cms.pichitmedev.com
1329 run('cp /home/phme/pichit.me/phme_faraday/configs/prod/cms/pickit/nginx.cms '
1330 '/home/phme/config/nginx.cms')
1331 # Link vassal in emperor
1332 if not exists('/etc/uwsgi-emperor/vassals/uwsgi.cms.ini'):
1333 with cd('/etc/uwsgi-emperor/vassals'):
1334 sudo('ln -s /home/phme/config/uwsgi.cms.ini')
1335 # Link nginx
1336 if not exists('/etc/nginx/sites-enabled/nginx.cms'):
1337 with cd('/etc/nginx/sites-enabled'):
1338 sudo('ln -s /home/phme/config/nginx.cms')
1339 run('touch /home/phme/config/uwsgi.cms.ini')
1340 sudo('service nginx restart')
1341 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py syncdb '
1342 '--settings cms_phme.settings.live')
1343 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py migrate '
1344 '--settings cms_phme.settings.live')
1345 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py collectstatic '
1346 '--settings cms_phme.settings.live')
1347 run('cp /home/phme/pichit.me/phme_faraday/configs/prod/cms/pickit/nginx.pickit.com '
1348 '/home/phme/config/nginx.pickit.com')
1349 sudo('service nginx restart')
1350
1351
1352@task
1353@hosts(*web_servers)
1354def deploy_live_cms(deployment_name=''):
1355 """
1356 Deploy to live CMS serves
1357
1358 :return:
1359 """
1360 if deployment_name:
1361 deployment_name = '"' + deployment_name + '" '
1362 publish_slack_message(u'[{}] Start CMS deployment {} [{}] ...'.format(
1363 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
1364 deployment_name, webs_human.get(env.host_string, 'undefined')), channel='systemlog')
1365 with cd('/home/phme/phme_cms'):
1366 run('git fetch')
1367 run('git pull origin master')
1368 run('/home/phme/envs/phme_cms_env/bin/pip install -r /home/phme/phme_cms/requirements.txt')
1369 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py syncdb '
1370 '--settings cms_phme.settings.live')
1371 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py migrate '
1372 '--settings cms_phme.settings.live')
1373 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py collectstatic '
1374 '--settings cms_phme.settings.live')
1375 if exists('/home/phme/config/uwsgi.cms.ini'):
1376 run('touch /home/phme/config/uwsgi.cms.ini')
1377 else:
1378 run('cp /home/phme/config/uwsgi.cms-src.ini /home/phme/config/uwsgi.cms.ini')
1379 publish_slack_message(u'[{}] Deployed CMS [{}]'.format(
1380 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
1381 webs_human.get(env.host_string, 'undefined')), channel='systemlog-dev')
1382
1383
1384@task
1385@hosts(*web_servers_pickit)
1386def deploy_live_cms_pickit(deployment_name=''):
1387 """
1388 Deploy to pickit live servers
1389
1390 :return:
1391 """
1392 if deployment_name:
1393 deployment_name = '"' + deployment_name + '" '
1394 publish_slack_message(u'[{}] Start CMS deployment {} [{}] pickit ...'.format(
1395 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
1396 deployment_name, webs_human.get(env.host_string, 'undefined')), channel='systemlog')
1397 with cd('/home/phme/phme_cms'):
1398 run('git fetch')
1399 run('git pull origin master')
1400 run('/home/phme/envs/phme_cms_env/bin/pip install -r /home/phme/phme_cms/requirements.txt')
1401 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py syncdb '
1402 '--settings cms_phme.settings.live_pickit')
1403 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py migrate '
1404 '--settings cms_phme.settings.live_pickit')
1405 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py collectstatic '
1406 '--settings cms_phme.settings.live_pickit')
1407 if exists('/home/phme/config/uwsgi.cms.ini'):
1408 run('touch /home/phme/config/uwsgi.cms.ini')
1409 else:
1410 run('cp /home/phme/config/uwsgi.cms-src.ini /home/phme/config/uwsgi.cms.ini')
1411 publish_slack_message(u'[{}] Deployed CMS [{}] pickit'.format(
1412 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
1413 webs_human.get(env.host_string, 'undefined')), channel='systemlog-dev')
1414
1415
1416@task
1417@hosts(*dev_web_servers)
1418def deploy_dev_cms(deployment_name=''):
1419 """
1420 Deploy CMS at development
1421
1422 :return:
1423 """
1424 if deployment_name:
1425 deployment_name = '"' + deployment_name + '" '
1426 publish_slack_message(u'[{}] Start development CMS deployment {} ...'.format(
1427 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
1428 deployment_name), channel='systemlog-dev')
1429 with cd('/home/phme/phme_cms'):
1430 run('git fetch')
1431 run('git pull origin master')
1432 run('/home/phme/envs/phme_cms_env/bin/pip install -r /home/phme/phme_cms/requirements.txt')
1433 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py syncdb '
1434 '--settings cms_phme.settings.dev')
1435 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py migrate '
1436 '--settings cms_phme.settings.dev')
1437 run('/home/phme/envs/phme_cms_env/bin/python /home/phme/phme_cms/manage.py collectstatic '
1438 '--settings cms_phme.settings.dev --noinput')
1439 if exists('/home/phme/config/uwsgi.cms.ini'):
1440 run('touch /home/phme/config/uwsgi.cms.pichitmedev.com.ini')
1441 else:
1442 run('cp /home/phme/config/uwsgi.cms.pichitmedev.com-src.ini '
1443 '/home/phme/config/uwsgi.cms.pichitmedev.com.ini')
1444 publish_slack_message(u'[{}] Deployed development CMS [{}]'.format(
1445 datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
1446 'phme-dev'), channel='systemlog-dev')