· 5 years ago · Feb 12, 2020, 10:00 PM
1# encoding: utf-8
2
3from __future__ import print_function
4
5import collections
6import csv
7import multiprocessing as mp
8import os
9import datetime
10import sys
11from pprint import pprint
12import re
13import itertools
14import json
15import logging
16from optparse import OptionConflictError
17import traceback
18
19from six import text_type
20from six.moves import input, xrange
21from six.moves.urllib.error import HTTPError
22from six.moves.urllib.parse import urljoin, urlparse
23from six.moves.urllib.request import urlopen
24
25import sqlalchemy as sa
26
27import routes
28import paste.script
29from paste.registry import Registry
30from paste.script.util.logging_config import fileConfig
31import click
32from ckan.cli import load_config as _get_config
33
34from ckan.config.middleware import make_app
35import ckan.logic as logic
36import ckan.model as model
37import ckan.include.rjsmin as rjsmin
38import ckan.include.rcssmin as rcssmin
39import ckan.plugins as p
40from ckan.common import config
41from ckan.common import asbool
42# This is a test Flask request context to be used internally.
43# Do not use it!
44_cli_test_request_context = None
45
46
47# NB No CKAN imports are allowed until after the config file is loaded.
48# i.e. do the imports in methods, after _load_config is called.
49# Otherwise loggers get disabled.
50
51
52def deprecation_warning(message=None):
53 '''
54 Print a deprecation warning to STDERR.
55
56 If ``message`` is given it is also printed to STDERR.
57 '''
58 sys.stderr.write(u'WARNING: This function is deprecated.')
59 if message:
60 sys.stderr.write(u' ' + message.strip())
61 sys.stderr.write(u'\n')
62
63
64def error(msg):
65 '''
66 Print an error message to STDOUT and exit with return code 1.
67 '''
68 sys.stderr.write(msg)
69 if not msg.endswith('\n'):
70 sys.stderr.write('\n')
71 sys.exit(1)
72
73
74def _parse_db_config(config_key=u'sqlalchemy.url'):
75 db_config = model.parse_db_config(config_key)
76 if not db_config:
77 raise Exception(
78 u'Could not extract db details from url: %r' % config[config_key]
79 )
80 return db_config
81
82
83def user_add(args):
84 '''Add new user if we use paster sysadmin add
85 or paster user add
86 '''
87 if len(args) < 1:
88 error('Error: you need to specify the user name.')
89 username = args[0]
90
91 # parse args into data_dict
92 data_dict = {'name': username}
93 for arg in args[1:]:
94 try:
95 field, value = arg.split('=', 1)
96 if field == 'sysadmin':
97 value = asbool(value)
98 data_dict[field] = value
99 except ValueError:
100 raise ValueError(
101 'Could not parse arg: %r (expected "<option>=<value>)"' % arg
102 )
103
104 # Required
105 while '@' not in data_dict.get('email', ''):
106 print('Error: Invalid email address')
107 data_dict['email'] = input('Email address: ').strip()
108
109 if 'password' not in data_dict:
110 data_dict['password'] = UserCmd.password_prompt()
111
112 # Optional
113 if 'fullname' in data_dict:
114 data_dict['fullname'] = data_dict['fullname'].decode(
115 sys.getfilesystemencoding()
116 )
117
118 print('Creating user: %r' % username)
119
120 try:
121 import ckan.logic as logic
122 import ckan.model as model
123 site_user = logic.get_action('get_site_user')({
124 'model': model,
125 'ignore_auth': True},
126 {}
127 )
128 context = {
129 'model': model,
130 'session': model.Session,
131 'ignore_auth': True,
132 'user': site_user['name'],
133 }
134 user_dict = logic.get_action('user_create')(context, data_dict)
135 pprint(user_dict)
136 except logic.ValidationError as e:
137 error(traceback.format_exc())
138
139## from http://code.activestate.com/recipes/577058/ MIT licence.
140## Written by Trent Mick
141def query_yes_no(question, default="yes"):
142 """Ask a yes/no question via input() and return their answer.
143
144 "question" is a string that is presented to the user.
145 "default" is the presumed answer if the user just hits <Enter>.
146 It must be "yes" (the default), "no" or None (meaning
147 an answer is required of the user).
148
149 The "answer" return value is one of "yes" or "no".
150 """
151 valid = {"yes": "yes", "y": "yes", "ye": "yes",
152 "no": "no", "n": "no"}
153 if default is None:
154 prompt = " [y/n] "
155 elif default == "yes":
156 prompt = " [Y/n] "
157 elif default == "no":
158 prompt = " [y/N] "
159 else:
160 raise ValueError("invalid default answer: '%s'" % default)
161
162 while 1:
163 sys.stdout.write(question + prompt)
164 choice = input().strip().lower()
165 if default is not None and choice == '':
166 return default
167 elif choice in valid.keys():
168 return valid[choice]
169 else:
170 sys.stdout.write("Please respond with 'yes' or 'no' "
171 "(or 'y' or 'n').\n")
172
173
174class MockTranslator(object):
175 def gettext(self, value):
176 return value
177
178 def ugettext(self, value):
179 return value
180
181 def ungettext(self, singular, plural, n):
182 if n > 1:
183 return plural
184 return singular
185
186
187def load_config(config, load_site_user=True):
188 conf = _get_config(config)
189 assert 'ckan' not in dir() # otherwise loggers would be disabled
190 # We have now loaded the config. Now we can import ckan for the
191 # first time.
192 from ckan.config.environment import load_environment
193 load_environment(conf.global_conf, conf.local_conf)
194
195 # Set this internal test request context with the configured environment so
196 # it can be used when calling url_for from the CLI.
197 global _cli_test_request_context
198
199 app = make_app(conf.global_conf, **conf.local_conf)
200 flask_app = app.apps['flask_app']._wsgi_app
201 _cli_test_request_context = flask_app.test_request_context()
202
203 registry = Registry()
204 registry.prepare()
205 import pylons
206 registry.register(pylons.translator, MockTranslator())
207
208 site_user = None
209 if model.user_table.exists() and load_site_user:
210 # If the DB has already been initialized, create and register
211 # a pylons context object, and add the site user to it, so the
212 # auth works as in a normal web request
213 c = pylons.util.AttribSafeContextObj()
214
215 registry.register(pylons.c, c)
216
217 site_user = logic.get_action('get_site_user')({'ignore_auth': True}, {})
218
219 pylons.c.user = site_user['name']
220 pylons.c.userobj = model.User.get(site_user['name'])
221
222 ## give routes enough information to run url_for
223 parsed = urlparse(conf.local_conf.get('ckan.site_url', 'http://0.0.0.0'))
224 request_config = routes.request_config()
225 request_config.host = parsed.netloc + parsed.path
226 request_config.protocol = parsed.scheme
227
228 return site_user
229
230
231def paster_click_group(summary):
232 '''Return a paster command click.Group for paster subcommands
233
234 :param command: the paster command linked to this function from
235 setup.py, used in help text (e.g. "datastore")
236 :param summary: summary text used in paster's help/command listings
237 (e.g. "Perform commands to set up the datastore")
238 '''
239 class PasterClickGroup(click.Group):
240 '''A click.Group that may be called like a paster command'''
241 def __call__(self, ignored_command):
242 sys.argv.remove(ignored_command)
243 return super(PasterClickGroup, self).__call__(
244 prog_name=u'paster ' + ignored_command,
245 help_option_names=[u'-h', u'--help'],
246 obj={})
247
248 @click.group(cls=PasterClickGroup)
249 @click.option(
250 '--plugin',
251 metavar='ckan',
252 help='paster plugin (when run outside ckan directory)')
253 @click_config_option
254 @click.pass_context
255 def cli(ctx, plugin, config):
256 ctx.obj['config'] = config
257
258
259 cli.summary = summary
260 cli.group_name = u'ckan'
261 return cli
262
263
264# common definition for paster ... --config
265click_config_option = click.option(
266 '-c',
267 '--config',
268 default=None,
269 metavar='CONFIG',
270 help=u'Config file to use (default: development.ini)')
271
272
273class CkanCommand(paste.script.command.Command):
274 '''Base class for classes that implement CKAN paster commands to inherit.'''
275 parser = paste.script.command.Command.standard_parser(verbose=True)
276 parser.add_option('-c', '--config', dest='config',
277 help='Config file to use.')
278 parser.add_option('-f', '--file',
279 action='store',
280 dest='file_path',
281 help="File to dump results to (if needed)")
282 default_verbosity = 1
283 group_name = 'ckan'
284
285 def _load_config(self, load_site_user=True):
286 self.site_user = load_config(self.options.config, load_site_user)
287
288
289class ManageDb(CkanCommand):
290 '''Perform various tasks on the database.
291
292 db create - alias of db upgrade
293 db init - create and put in default data
294 db clean - clears db (including dropping tables) and
295 search index
296 db upgrade [version no.] - Data migrate
297 db version - returns current version of data schema
298 db create-from-model - create database from the model (indexes not made)
299 db migrate-filestore - migrate all uploaded data from the 2.1 filesore.
300 '''
301 summary = __doc__.split('\n')[0]
302 usage = __doc__
303 max_args = None
304 min_args = 1
305
306 def command(self):
307 cmd = self.args[0]
308
309 self._load_config(cmd != 'upgrade')
310
311 import ckan.model as model
312 import ckan.lib.search as search
313
314 if cmd == 'init':
315
316 model.repo.init_db()
317 if self.verbose:
318 print('Initialising DB: SUCCESS')
319 elif cmd == 'clean' or cmd == 'drop':
320
321 # remove any *.pyc version files to prevent conflicts
322 v_path = os.path.join(os.path.dirname(__file__),
323 '..', 'migration', 'versions', '*.pyc')
324 import glob
325 filelist = glob.glob(v_path)
326 for f in filelist:
327 os.remove(f)
328
329 model.repo.clean_db()
330 search.clear_all()
331 if self.verbose:
332 print('Cleaning DB: SUCCESS')
333 elif cmd == 'upgrade':
334 model.repo.upgrade_db(*self.args[1:])
335 elif cmd == 'downgrade':
336 model.repo.downgrade_db(*self.args[1:])
337 elif cmd == 'version':
338 self.version()
339 elif cmd == 'create-from-model':
340 model.repo.create_db()
341 if self.verbose:
342 print('Creating DB: SUCCESS')
343 else:
344 error('Command %s not recognized' % cmd)
345
346 def version(self):
347 from ckan.model import Session
348 print(Session.execute('select version from '
349 'migrate_version;').fetchall())
350
351
352class SearchIndexCommand(CkanCommand):
353 '''Creates a search index for all datasets
354
355 Usage:
356 search-index [-i] [-o] [-r] [-e] [-q] rebuild [dataset_name] - reindex dataset_name if given, if not then rebuild
357 full search index (all datasets)
358 search-index rebuild_fast - reindex using multiprocessing using all cores.
359 This acts in the same way as rubuild -r [EXPERIMENTAL]
360 search-index check - checks for datasets not indexed
361 search-index show DATASET_NAME - shows index of a dataset
362 search-index clear [dataset_name] - clears the search index for the provided dataset or
363 for the whole ckan instance
364 '''
365
366 summary = __doc__.split('\n')[0]
367 usage = __doc__
368 max_args = 2
369 min_args = 0
370
371 def __init__(self, name):
372 super(SearchIndexCommand, self).__init__(name)
373
374 self.parser.add_option('-i', '--force', dest='force',
375 action='store_true', default=False,
376 help='Ignore exceptions when rebuilding the index')
377
378 self.parser.add_option('-o', '--only-missing', dest='only_missing',
379 action='store_true', default=False,
380 help='Index non indexed datasets only')
381
382 self.parser.add_option('-r', '--refresh', dest='refresh',
383 action='store_true', default=False,
384 help='Refresh current index (does not clear the existing one)')
385
386 self.parser.add_option('-q', '--quiet', dest='quiet',
387 action='store_true', default=False,
388 help='Do not output index rebuild progress')
389
390 self.parser.add_option('-e', '--commit-each', dest='commit_each',
391 action='store_true', default=False, help=
392'''Perform a commit after indexing each dataset. This ensures that changes are
393immediately available on the search, but slows significantly the process.
394Default is false.''')
395
396 def command(self):
397 if not self.args:
398 # default to printing help
399 print(self.usage)
400 return
401
402 cmd = self.args[0]
403 # Do not run load_config yet
404 if cmd == 'rebuild_fast':
405 self.rebuild_fast()
406 return
407
408 self._load_config()
409 if cmd == 'rebuild':
410 self.rebuild()
411 elif cmd == 'check':
412 self.check()
413 elif cmd == 'show':
414 self.show()
415 elif cmd == 'clear':
416 self.clear()
417 else:
418 print('Command %s not recognized' % cmd)
419
420 def rebuild(self):
421 from ckan.lib.search import rebuild, commit
422
423 # BY default we don't commit after each request to Solr, as it is
424 # a really heavy operation and slows things a lot
425
426 if len(self.args) > 1:
427 rebuild(self.args[1])
428 else:
429 rebuild(only_missing=self.options.only_missing,
430 force=self.options.force,
431 refresh=self.options.refresh,
432 defer_commit=(not self.options.commit_each),
433 quiet=self.options.quiet)
434
435 if not self.options.commit_each:
436 commit()
437
438 def check(self):
439 from ckan.lib.search import check
440 check()
441
442 def show(self):
443 from ckan.lib.search import show
444
445 if not len(self.args) == 2:
446 print('Missing parameter: dataset-name')
447 return
448 index = show(self.args[1])
449 pprint(index)
450
451 def clear(self):
452 from ckan.lib.search import clear, clear_all
453 package_id = self.args[1] if len(self.args) > 1 else None
454 if not package_id:
455 clear_all()
456 else:
457 clear(package_id)
458
459 def rebuild_fast(self):
460 ### Get out config but without starting pylons environment ####
461 conf = _get_config()
462
463 ### Get ids using own engine, otherwise multiprocess will balk
464 db_url = conf['sqlalchemy.url']
465 engine = sa.create_engine(db_url)
466 package_ids = []
467 result = engine.execute("select id from package where state = 'active';")
468 for row in result:
469 package_ids.append(row[0])
470
471 def start(ids):
472 ## load actual enviroment for each subprocess, so each have thier own
473 ## sa session
474 self._load_config()
475 from ckan.lib.search import rebuild, commit
476 rebuild(package_ids=ids)
477 commit()
478
479 def chunks(l, n):
480 """ Yield n successive chunks from l.
481 """
482 newn = int(len(l) / n)
483 for i in xrange(0, n-1):
484 yield l[i*newn:i*newn+newn]
485 yield l[n*newn-newn:]
486
487 processes = []
488 for chunk in chunks(package_ids, mp.cpu_count()):
489 process = mp.Process(target=start, args=(chunk,))
490 processes.append(process)
491 process.daemon = True
492 process.start()
493
494 for process in processes:
495 process.join()
496
497
498class Notification(CkanCommand):
499 '''Send out modification notifications.
500
501 In "replay" mode, an update signal is sent for each dataset in the database.
502
503 Usage:
504 notify replay - send out modification signals
505 '''
506
507 summary = __doc__.split('\n')[0]
508 usage = __doc__
509 max_args = 1
510 min_args = 0
511
512 def command(self):
513 self._load_config()
514 from ckan.model import Session, Package, DomainObjectOperation
515 from ckan.model.modification import DomainObjectModificationExtension
516
517 if not self.args:
518 # default to run
519 cmd = 'replay'
520 else:
521 cmd = self.args[0]
522
523 if cmd == 'replay':
524 dome = DomainObjectModificationExtension()
525 for package in Session.query(Package):
526 dome.notify(package, DomainObjectOperation.changed)
527 else:
528 print('Command %s not recognized' % cmd)
529
530
531class RDFExport(CkanCommand):
532 '''Export active datasets as RDF
533 This command dumps out all currently active datasets as RDF into the
534 specified folder.
535
536 Usage:
537 paster rdf-export /path/to/store/output
538 '''
539 summary = __doc__.split('\n')[0]
540 usage = __doc__
541
542 def command(self):
543 self._load_config()
544
545 if not self.args:
546 # default to run
547 print(RDFExport.__doc__)
548 else:
549 self.export_datasets(self.args[0])
550
551 def export_datasets(self, out_folder):
552 '''
553 Export datasets as RDF to an output folder.
554 '''
555 from ckan.common import config
556 import ckan.model as model
557 import ckan.logic as logic
558 import ckan.lib.helpers as h
559
560 # Create output folder if not exists
561 if not os.path.isdir(out_folder):
562 os.makedirs(out_folder)
563
564 fetch_url = config['ckan.site_url']
565 user = logic.get_action('get_site_user')({'model': model, 'ignore_auth': True}, {})
566 context = {'model': model, 'session': model.Session, 'user': user['name']}
567 dataset_names = logic.get_action('package_list')(context, {})
568 for dataset_name in dataset_names:
569 dd = logic.get_action('package_show')(context, {'id': dataset_name})
570 if not dd['state'] == 'active':
571 continue
572
573 url = h.url_for('dataset.read', id=dd['name'])
574
575 url = urljoin(fetch_url, url[1:]) + '.rdf'
576 try:
577 fname = os.path.join(out_folder, dd['name']) + ".rdf"
578 try:
579 r = urlopen(url).read()
580 except HTTPError as e:
581 if e.code == 404:
582 error('Please install ckanext-dcat and enable the ' +
583 '`dcat` plugin to use the RDF serializations')
584 with open(fname, 'wb') as f:
585 f.write(r)
586 except IOError as ioe:
587 sys.stderr.write(str(ioe) + "\n")
588
589
590class Sysadmin(CkanCommand):
591 '''Gives sysadmin rights to a named user
592
593 Usage:
594 sysadmin - lists sysadmins
595 sysadmin list - lists sysadmins
596 sysadmin add USERNAME - make an existing user into a sysadmin
597 sysadmin add USERNAME [FIELD1=VALUE1 FIELD2=VALUE2 ...]
598 - creates a new user that is a sysadmin
599 (prompts for password and email if not
600 supplied).
601 Field can be: apikey
602 email
603 fullname
604 name (this will be the username)
605 password
606 sysadmin remove USERNAME - removes user from sysadmins
607 '''
608
609 summary = __doc__.split('\n')[0]
610 usage = __doc__
611 max_args = None
612 min_args = 0
613
614 def command(self):
615 self._load_config()
616
617 cmd = self.args[0] if self.args else None
618 if cmd is None or cmd == 'list':
619 self.list()
620 elif cmd == 'add':
621 self.add()
622 elif cmd == 'remove':
623 self.remove()
624 else:
625 print('Command %s not recognized' % cmd)
626
627 def list(self):
628 import ckan.model as model
629 print('Sysadmins:')
630 sysadmins = model.Session.query(model.User).filter_by(sysadmin=True,
631 state='active')
632 print('count = %i' % sysadmins.count())
633 for sysadmin in sysadmins:
634 print('%s name=%s email=%s id=%s' % (
635 sysadmin.__class__.__name__,
636 sysadmin.name,
637 sysadmin.email,
638 sysadmin.id))
639
640 def add(self):
641 import ckan.model as model
642
643 if len(self.args) < 2:
644 print('Need name of the user to be made sysadmin.')
645 return
646 username = self.args[1]
647
648 user = model.User.by_name(text_type(username))
649 if not user:
650 print('User "%s" not found' % username)
651 makeuser = input('Create new user: %s? [y/n]' % username)
652 if makeuser == 'y':
653 user_add(self.args[1:])
654 user = model.User.by_name(text_type(username))
655 else:
656 print('Exiting ...')
657 return
658
659 user.sysadmin = True
660 model.Session.add(user)
661 model.repo.commit_and_remove()
662 print('Added %s as sysadmin' % username)
663
664 def remove(self):
665 import ckan.model as model
666
667 if len(self.args) < 2:
668 print('Need name of the user to be made sysadmin.')
669 return
670 username = self.args[1]
671
672 user = model.User.by_name(text_type(username))
673 if not user:
674 print('Error: user "%s" not found!' % username)
675 return
676 user.sysadmin = False
677 model.repo.commit_and_remove()
678
679
680class UserCmd(CkanCommand):
681 '''Manage users
682
683 Usage:
684 user - lists users
685 user list - lists users
686 user USERNAME - shows user properties
687 user add USERNAME [FIELD1=VALUE1 FIELD2=VALUE2 ...]
688 - add a user (prompts for email and
689 password if not supplied).
690 Field can be: apikey
691 email
692 fullname
693 name (this will be the username)
694 password
695 user setpass USERNAME - set user password (prompts)
696 user remove USERNAME - removes user from users
697 user search QUERY - searches for a user name
698 '''
699 summary = __doc__.split('\n')[0]
700 usage = __doc__
701 max_args = None
702 min_args = 0
703
704 def command(self):
705 self._load_config()
706
707 if not self.args:
708 self.list()
709 else:
710 cmd = self.args[0]
711 if cmd == 'add':
712 self.add()
713 elif cmd == 'remove':
714 self.remove()
715 elif cmd == 'search':
716 self.search()
717 elif cmd == 'setpass':
718 self.setpass()
719 elif cmd == 'list':
720 self.list()
721 else:
722 self.show()
723
724 def get_user_str(self, user):
725 user_str = 'name=%s' % user.name
726 if user.name != user.display_name:
727 user_str += ' display=%s' % user.display_name
728 return user_str
729
730 def list(self):
731 import ckan.model as model
732 print('Users:')
733 users = model.Session.query(model.User).filter_by(state='active')
734 print('count = %i' % users.count())
735 for user in users:
736 print(self.get_user_str(user))
737
738 def show(self):
739 import ckan.model as model
740
741 username = self.args[0]
742 user = model.User.get(text_type(username))
743 print('User: \n', user)
744
745 def setpass(self):
746 import ckan.model as model
747
748 if len(self.args) < 2:
749 print('Need name of the user.')
750 return
751 username = self.args[1]
752 user = model.User.get(username)
753 print('Editing user: %r' % user.name)
754
755 password = self.password_prompt()
756 user.password = password
757 model.repo.commit_and_remove()
758 print('Done')
759
760 def search(self):
761 import ckan.model as model
762
763 if len(self.args) < 2:
764 print('Need user name query string.')
765 return
766 query_str = self.args[1]
767
768 query = model.User.search(query_str)
769 print('%i users matching %r:' % (query.count(), query_str))
770 for user in query.all():
771 print(self.get_user_str(user))
772
773 @classmethod
774 def password_prompt(cls):
775 import getpass
776 password1 = None
777 while not password1:
778 password1 = getpass.getpass('Password: ')
779 password2 = getpass.getpass('Confirm password: ')
780 if password1 != password2:
781 error('Passwords do not match')
782 return password1
783
784 def add(self):
785 user_add(self.args[1:])
786
787 def remove(self):
788 import ckan.model as model
789
790 if len(self.args) < 2:
791 print('Need name of the user.')
792 return
793 username = self.args[1]
794
795 p.toolkit.get_action('user_delete')(
796 {'model': model, 'ignore_auth': True},
797 {'id': username})
798 print('Deleted user: %s' % username)
799
800
801class DatasetCmd(CkanCommand):
802 '''Manage datasets
803
804 Usage:
805 dataset DATASET_NAME|ID - shows dataset properties
806 dataset show DATASET_NAME|ID - shows dataset properties
807 dataset list - lists datasets
808 dataset delete [DATASET_NAME|ID] - changes dataset state to 'deleted'
809 dataset purge [DATASET_NAME|ID] - removes dataset from db entirely
810 '''
811 summary = __doc__.split('\n')[0]
812 usage = __doc__
813 max_args = 3
814 min_args = 0
815
816 def command(self):
817 self._load_config()
818
819 if not self.args:
820 print(self.usage)
821 else:
822 cmd = self.args[0]
823 if cmd == 'delete':
824 self.delete(self.args[1])
825 elif cmd == 'purge':
826 self.purge(self.args[1])
827 elif cmd == 'list':
828 self.list()
829 elif cmd == 'show':
830 self.show(self.args[1])
831 else:
832 self.show(self.args[0])
833
834 def list(self):
835 import ckan.model as model
836 print('Datasets:')
837 datasets = model.Session.query(model.Package)
838 print('count = %i' % datasets.count())
839 for dataset in datasets:
840 state = ('(%s)' % dataset.state) if dataset.state != 'active' else ''
841 print('%s %s %s' % (dataset.id, dataset.name, state))
842
843 def _get_dataset(self, dataset_ref):
844 import ckan.model as model
845 dataset = model.Package.get(text_type(dataset_ref))
846 assert dataset, 'Could not find dataset matching reference: %r' % dataset_ref
847 return dataset
848
849 def show(self, dataset_ref):
850 import pprint
851 dataset = self._get_dataset(dataset_ref)
852 pprint.pprint(dataset.as_dict())
853
854 def delete(self, dataset_ref):
855 import ckan.model as model
856 dataset = self._get_dataset(dataset_ref)
857 old_state = dataset.state
858
859 dataset.delete()
860 model.repo.commit_and_remove()
861 dataset = self._get_dataset(dataset_ref)
862 print('%s %s -> %s' % (dataset.name, old_state, dataset.state))
863
864 def purge(self, dataset_ref):
865 import ckan.logic as logic
866 dataset = self._get_dataset(dataset_ref)
867 name = dataset.name
868
869 site_user = logic.get_action('get_site_user')({'ignore_auth': True}, {})
870 context = {'user': site_user['name']}
871 logic.get_action('dataset_purge')(
872 context, {'id': dataset_ref})
873 print('%s purged' % name)
874
875
876class Ratings(CkanCommand):
877 '''Manage the ratings stored in the db
878
879 Usage:
880 ratings count - counts ratings
881 ratings clean - remove all ratings
882 ratings clean-anonymous - remove only anonymous ratings
883 '''
884
885 summary = __doc__.split('\n')[0]
886 usage = __doc__
887 max_args = 1
888 min_args = 1
889
890 def command(self):
891 self._load_config()
892 import ckan.model as model
893
894 cmd = self.args[0]
895 if cmd == 'count':
896 self.count()
897 elif cmd == 'clean':
898 self.clean()
899 elif cmd == 'clean-anonymous':
900 self.clean(user_ratings=False)
901 else:
902 print('Command %s not recognized' % cmd)
903
904 def count(self):
905 import ckan.model as model
906 q = model.Session.query(model.Rating)
907 print("%i ratings" % q.count())
908 q = q.filter(model.Rating.user_id is None)
909 print("of which %i are anonymous ratings" % q.count())
910
911 def clean(self, user_ratings=True):
912 import ckan.model as model
913 q = model.Session.query(model.Rating)
914 print("%i ratings" % q.count())
915 if not user_ratings:
916 q = q.filter(model.Rating.user_id is None)
917 print("of which %i are anonymous ratings" % q.count())
918 ratings = q.all()
919 for rating in ratings:
920 rating.purge()
921 model.repo.commit_and_remove()
922
923
924## Used by the Tracking class
925_ViewCount = collections.namedtuple("ViewCount", "id name count")
926
927
928class Tracking(CkanCommand):
929 '''Update tracking statistics
930
931 Usage:
932 tracking update [start_date] - update tracking stats
933 tracking export FILE [start_date] - export tracking stats to a csv file
934 '''
935
936 summary = __doc__.split('\n')[0]
937 usage = __doc__
938 max_args = 3
939 min_args = 1
940
941 def command(self):
942 self._load_config()
943 import ckan.model as model
944 engine = model.meta.engine
945
946 cmd = self.args[0]
947 if cmd == 'update':
948 start_date = self.args[1] if len(self.args) > 1 else None
949 self.update_all(engine, start_date)
950 elif cmd == 'export':
951 if len(self.args) <= 1:
952 error(self.__class__.__doc__)
953 output_file = self.args[1]
954 start_date = self.args[2] if len(self.args) > 2 else None
955 self.update_all(engine, start_date)
956 self.export_tracking(engine, output_file)
957 else:
958 error(self.__class__.__doc__)
959
960 def update_all(self, engine, start_date=None):
961 if start_date:
962 start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
963 else:
964 # No date given. See when we last have data for and get data
965 # from 2 days before then in case new data is available.
966 # If no date here then use 2011-01-01 as the start date
967 sql = '''SELECT tracking_date from tracking_summary
968 ORDER BY tracking_date DESC LIMIT 1;'''
969 result = engine.execute(sql).fetchall()
970 if result:
971 start_date = result[0]['tracking_date']
972 start_date += datetime.timedelta(-2)
973 # convert date to datetime
974 combine = datetime.datetime.combine
975 start_date = combine(start_date, datetime.time(0))
976 else:
977 start_date = datetime.datetime(2011, 1, 1)
978 start_date_solrsync = start_date
979 end_date = datetime.datetime.now()
980
981 while start_date < end_date:
982 stop_date = start_date + datetime.timedelta(1)
983 self.update_tracking(engine, start_date)
984 print('tracking updated for %s' % start_date)
985 start_date = stop_date
986
987 self.update_tracking_solr(engine, start_date_solrsync)
988
989 def _total_views(self, engine):
990 sql = '''
991 SELECT p.id,
992 p.name,
993 COALESCE(SUM(s.count), 0) AS total_views
994 FROM package AS p
995 LEFT OUTER JOIN tracking_summary AS s ON s.package_id = p.id
996 GROUP BY p.id, p.name
997 ORDER BY total_views DESC
998 '''
999 return [_ViewCount(*t) for t in engine.execute(sql).fetchall()]
1000
1001 def _recent_views(self, engine, measure_from):
1002 sql = '''
1003 SELECT p.id,
1004 p.name,
1005 COALESCE(SUM(s.count), 0) AS total_views
1006 FROM package AS p
1007 LEFT OUTER JOIN tracking_summary AS s ON s.package_id = p.id
1008 WHERE s.tracking_date >= %(measure_from)s
1009 GROUP BY p.id, p.name
1010 ORDER BY total_views DESC
1011 '''
1012 return [_ViewCount(*t) for t in engine.execute(sql, measure_from=str(measure_from)).fetchall()]
1013
1014 def export_tracking(self, engine, output_filename):
1015 '''Write tracking summary to a csv file.'''
1016 HEADINGS = [
1017 "dataset id",
1018 "dataset name",
1019 "total views",
1020 "recent views (last 2 weeks)",
1021 ]
1022
1023 measure_from = datetime.date.today() - datetime.timedelta(days=14)
1024 recent_views = self._recent_views(engine, measure_from)
1025 total_views = self._total_views(engine)
1026
1027 with open(output_filename, 'w') as fh:
1028 f_out = csv.writer(fh)
1029 f_out.writerow(HEADINGS)
1030 recent_views_for_id = dict((r.id, r.count) for r in recent_views)
1031 f_out.writerows([(r.id,
1032 r.name,
1033 r.count,
1034 recent_views_for_id.get(r.id, 0))
1035 for r in total_views])
1036
1037 def update_tracking(self, engine, summary_date):
1038 PACKAGE_URL = '/dataset/'
1039 # clear out existing data before adding new
1040 sql = '''DELETE FROM tracking_summary
1041 WHERE tracking_date='%s'; ''' % summary_date
1042 engine.execute(sql)
1043
1044 sql = '''SELECT DISTINCT url, user_key,
1045 CAST(access_timestamp AS Date) AS tracking_date,
1046 tracking_type INTO tracking_tmp
1047 FROM tracking_raw
1048 WHERE CAST(access_timestamp as Date)=%s;
1049
1050 INSERT INTO tracking_summary
1051 (url, count, tracking_date, tracking_type)
1052 SELECT url, count(user_key), tracking_date, tracking_type
1053 FROM tracking_tmp
1054 GROUP BY url, tracking_date, tracking_type;
1055
1056 DROP TABLE tracking_tmp;
1057 COMMIT;'''
1058 engine.execute(sql, summary_date)
1059
1060 # get ids for dataset urls
1061 sql = '''UPDATE tracking_summary t
1062 SET package_id = COALESCE(
1063 (SELECT id FROM package p
1064 WHERE p.name = regexp_replace(' ' || t.url, '^[ ]{1}(/\w{2}){0,1}' || %s, ''))
1065 ,'~~not~found~~')
1066 WHERE t.package_id IS NULL
1067 AND tracking_type = 'page';'''
1068 engine.execute(sql, PACKAGE_URL)
1069
1070 # update summary totals for resources
1071 sql = '''UPDATE tracking_summary t1
1072 SET running_total = (
1073 SELECT sum(count)
1074 FROM tracking_summary t2
1075 WHERE t1.url = t2.url
1076 AND t2.tracking_date <= t1.tracking_date
1077 )
1078 ,recent_views = (
1079 SELECT sum(count)
1080 FROM tracking_summary t2
1081 WHERE t1.url = t2.url
1082 AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14
1083 )
1084 WHERE t1.running_total = 0 AND tracking_type = 'resource';'''
1085 engine.execute(sql)
1086
1087 # update summary totals for pages
1088 sql = '''UPDATE tracking_summary t1
1089 SET running_total = (
1090 SELECT sum(count)
1091 FROM tracking_summary t2
1092 WHERE t1.package_id = t2.package_id
1093 AND t2.tracking_date <= t1.tracking_date
1094 )
1095 ,recent_views = (
1096 SELECT sum(count)
1097 FROM tracking_summary t2
1098 WHERE t1.package_id = t2.package_id
1099 AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14
1100 )
1101 WHERE t1.running_total = 0 AND tracking_type = 'page'
1102 AND t1.package_id IS NOT NULL
1103 AND t1.package_id != '~~not~found~~';'''
1104 engine.execute(sql)
1105
1106 def update_tracking_solr(self, engine, start_date):
1107 sql = '''SELECT package_id FROM tracking_summary
1108 where package_id!='~~not~found~~'
1109 and tracking_date >= %s;'''
1110 results = engine.execute(sql, start_date)
1111
1112 package_ids = set()
1113 for row in results:
1114 package_ids.add(row['package_id'])
1115
1116 total = len(package_ids)
1117 not_found = 0
1118 print('%i package index%s to be rebuilt starting from %s' % (total, '' if total < 2 else 'es', start_date))
1119
1120 from ckan.lib.search import rebuild
1121 for package_id in package_ids:
1122 try:
1123 rebuild(package_id)
1124 except logic.NotFound:
1125 print("Error: package %s not found." % (package_id))
1126 not_found += 1
1127 except KeyboardInterrupt:
1128 print("Stopped.")
1129 return
1130 except:
1131 raise
1132 print('search index rebuilding done.' + (' %i not found.' % (not_found) if not_found else ""))
1133
1134
1135class PluginInfo(CkanCommand):
1136 '''Provide info on installed plugins.
1137 '''
1138
1139 summary = __doc__.split('\n')[0]
1140 usage = __doc__
1141 max_args = 0
1142 min_args = 0
1143
1144 def command(self):
1145 self.get_info()
1146
1147 def get_info(self):
1148 ''' print info about current plugins from the .ini file'''
1149 import ckan.plugins as p
1150 self._load_config()
1151 interfaces = {}
1152 plugins = {}
1153 for name in dir(p):
1154 item = getattr(p, name)
1155 try:
1156 if issubclass(item, p.Interface):
1157 interfaces[item] = {'class': item}
1158 except TypeError:
1159 pass
1160
1161 for interface in interfaces:
1162 for plugin in p.PluginImplementations(interface):
1163 name = plugin.name
1164 if name not in plugins:
1165 plugins[name] = {'doc': plugin.__doc__,
1166 'class': plugin,
1167 'implements': []}
1168 plugins[name]['implements'].append(interface.__name__)
1169
1170 for plugin in plugins:
1171 p = plugins[plugin]
1172 print(plugin + ':')
1173 print('-' * (len(plugin) + 1))
1174 if p['doc']:
1175 print(p['doc'])
1176 print('Implements:')
1177 for i in p['implements']:
1178 extra = None
1179 if i == 'ITemplateHelpers':
1180 extra = self.template_helpers(p['class'])
1181 if i == 'IActions':
1182 extra = self.actions(p['class'])
1183 print(' %s' % i)
1184 if extra:
1185 print(extra)
1186 print()
1187
1188 def actions(self, cls):
1189 ''' Return readable action function info. '''
1190 actions = cls.get_actions()
1191 return self.function_info(actions)
1192
1193 def template_helpers(self, cls):
1194 ''' Return readable helper function info. '''
1195 helpers = cls.get_helpers()
1196 return self.function_info(helpers)
1197
1198 def function_info(self, functions):
1199 ''' Take a dict of functions and output readable info '''
1200 import inspect
1201 output = []
1202 for function_name in functions:
1203 fn = functions[function_name]
1204 args_info = inspect.getargspec(fn)
1205 params = args_info.args
1206 num_params = len(params)
1207 if args_info.varargs:
1208 params.append('*' + args_info.varargs)
1209 if args_info.keywords:
1210 params.append('**' + args_info.keywords)
1211 if args_info.defaults:
1212 offset = num_params - len(args_info.defaults)
1213 for i, v in enumerate(args_info.defaults):
1214 params[i + offset] = params[i + offset] + '=' + repr(v)
1215 # is this a classmethod if so remove the first parameter
1216 if inspect.ismethod(fn) and inspect.isclass(fn.__self__):
1217 params = params[1:]
1218 params = ', '.join(params)
1219 output.append(' %s(%s)' % (function_name, params))
1220 # doc string
1221 if fn.__doc__:
1222 bits = fn.__doc__.split('\n')
1223 for bit in bits:
1224 output.append(' %s' % bit)
1225 return ('\n').join(output)
1226
1227
1228class CreateTestDataCommand(CkanCommand):
1229 '''Create test data in the database.
1230 Tests can also delete the created objects easily with the delete() method.
1231
1232 create-test-data - annakarenina and warandpeace
1233 create-test-data search - realistic data to test search
1234 create-test-data gov - government style data
1235 create-test-data family - package relationships data
1236 create-test-data user - create a user 'tester' with api key 'tester'
1237 create-test-data translations - annakarenina, warandpeace, and some test
1238 translations of terms
1239 create-test-data vocabs - annakerenina, warandpeace, and some test
1240 vocabularies
1241 create-test-data hierarchy - hierarchy of groups
1242 '''
1243 summary = __doc__.split('\n')[0]
1244 usage = __doc__
1245 max_args = 1
1246 min_args = 0
1247
1248 def command(self):
1249 self._load_config()
1250 from ckan.lib.create_test_data import CreateTestData
1251
1252 if self.args:
1253 cmd = self.args[0]
1254 else:
1255 cmd = 'basic'
1256 if self.verbose:
1257 print('Creating %s test data' % cmd)
1258 if cmd == 'basic':
1259 CreateTestData.create_basic_test_data()
1260 elif cmd == 'user':
1261 CreateTestData.create_test_user()
1262 print('Created user %r with password %r and apikey %r' %
1263 ('tester', 'tester', 'tester'))
1264 elif cmd == 'search':
1265 CreateTestData.create_search_test_data()
1266 elif cmd == 'gov':
1267 CreateTestData.create_gov_test_data()
1268 elif cmd == 'family':
1269 CreateTestData.create_family_test_data()
1270 elif cmd == 'translations':
1271 CreateTestData.create_translations_test_data()
1272 elif cmd == 'vocabs':
1273 CreateTestData.create_vocabs_test_data()
1274 elif cmd == 'hierarchy':
1275 CreateTestData.create_group_hierarchy_test_data()
1276 else:
1277 print('Command %s not recognized' % cmd)
1278 raise NotImplementedError
1279 if self.verbose:
1280 print('Creating %s test data: Complete!' % cmd)
1281
1282
1283class Profile(CkanCommand):
1284 '''Code speed profiler
1285 Provide a ckan url and it will make the request and record
1286 how long each function call took in a file that can be read
1287 by pstats.Stats (command-line) or runsnakerun (gui).
1288
1289 Usage:
1290 profile URL [username]
1291
1292 e.g. profile /data/search
1293
1294 The result is saved in profile.data.search
1295 To view the profile in runsnakerun:
1296 runsnakerun ckan.data.search.profile
1297
1298 You may need to install python module: cProfile
1299 '''
1300 summary = __doc__.split('\n')[0]
1301 usage = __doc__
1302 max_args = 2
1303 min_args = 1
1304
1305 def _load_config_into_test_app(self):
1306 from paste.deploy import loadapp
1307 import paste.fixture
1308 if not self.options.config:
1309 msg = 'No config file supplied'
1310 raise self.BadCommand(msg)
1311 self.filename = os.path.abspath(self.options.config)
1312 if not os.path.exists(self.filename):
1313 raise AssertionError('Config filename %r does not exist.' % self.filename)
1314 fileConfig(self.filename)
1315
1316 wsgiapp = loadapp('config:' + self.filename)
1317 self.app = paste.fixture.TestApp(wsgiapp)
1318
1319 def command(self):
1320 self._load_config_into_test_app()
1321
1322 import paste.fixture
1323 import cProfile
1324 import re
1325
1326 url = self.args[0]
1327 if self.args[1:]:
1328 user = self.args[1]
1329 else:
1330 user = 'visitor'
1331
1332 def profile_url(url):
1333 try:
1334 res = self.app.get(url, status=[200],
1335 extra_environ={'REMOTE_USER': user})
1336 except paste.fixture.AppError:
1337 print('App error: ', url.strip())
1338 except KeyboardInterrupt:
1339 raise
1340 except Exception:
1341 error(traceback.format_exc())
1342
1343 output_filename = 'ckan%s.profile' % re.sub('[/?]', '.', url.replace('/', '.'))
1344 profile_command = "profile_url('%s')" % url
1345 cProfile.runctx(profile_command, globals(), locals(), filename=output_filename)
1346 import pstats
1347 stats = pstats.Stats(output_filename)
1348 stats.sort_stats('cumulative')
1349 stats.print_stats(0.1) # show only top 10% of lines
1350 print('Only top 10% of lines shown')
1351 print('Written profile to: %s' % output_filename)
1352
1353
1354class CreateColorSchemeCommand(CkanCommand):
1355 '''Create or remove a color scheme.
1356
1357 After running this, you'll need to regenerate the css files. See paster's less command for details.
1358
1359 color - creates a random color scheme
1360 color clear - clears any color scheme
1361 color <'HEX'> - uses as base color eg '#ff00ff' must be quoted.
1362 color <VALUE> - a float between 0.0 and 1.0 used as base hue
1363 color <COLOR_NAME> - html color name used for base color eg lightblue
1364 '''
1365
1366 summary = __doc__.split('\n')[0]
1367 usage = __doc__
1368 max_args = 1
1369 min_args = 0
1370
1371 rules = [
1372 '@layoutLinkColor',
1373 '@mastheadBackgroundColor',
1374 '@btnPrimaryBackground',
1375 '@btnPrimaryBackgroundHighlight',
1376 ]
1377
1378 # list of predefined colors
1379 color_list = {
1380 'aliceblue': '#f0fff8',
1381 'antiquewhite': '#faebd7',
1382 'aqua': '#00ffff',
1383 'aquamarine': '#7fffd4',
1384 'azure': '#f0ffff',
1385 'beige': '#f5f5dc',
1386 'bisque': '#ffe4c4',
1387 'black': '#000000',
1388 'blanchedalmond': '#ffebcd',
1389 'blue': '#0000ff',
1390 'blueviolet': '#8a2be2',
1391 'brown': '#a52a2a',
1392 'burlywood': '#deb887',
1393 'cadetblue': '#5f9ea0',
1394 'chartreuse': '#7fff00',
1395 'chocolate': '#d2691e',
1396 'coral': '#ff7f50',
1397 'cornflowerblue': '#6495ed',
1398 'cornsilk': '#fff8dc',
1399 'crimson': '#dc143c',
1400 'cyan': '#00ffff',
1401 'darkblue': '#00008b',
1402 'darkcyan': '#008b8b',
1403 'darkgoldenrod': '#b8860b',
1404 'darkgray': '#a9a9a9',
1405 'darkgrey': '#a9a9a9',
1406 'darkgreen': '#006400',
1407 'darkkhaki': '#bdb76b',
1408 'darkmagenta': '#8b008b',
1409 'darkolivegreen': '#556b2f',
1410 'darkorange': '#ff8c00',
1411 'darkorchid': '#9932cc',
1412 'darkred': '#8b0000',
1413 'darksalmon': '#e9967a',
1414 'darkseagreen': '#8fbc8f',
1415 'darkslateblue': '#483d8b',
1416 'darkslategray': '#2f4f4f',
1417 'darkslategrey': '#2f4f4f',
1418 'darkturquoise': '#00ced1',
1419 'darkviolet': '#9400d3',
1420 'deeppink': '#ff1493',
1421 'deepskyblue': '#00bfff',
1422 'dimgray': '#696969',
1423 'dimgrey': '#696969',
1424 'dodgerblue': '#1e90ff',
1425 'firebrick': '#b22222',
1426 'floralwhite': '#fffaf0',
1427 'forestgreen': '#228b22',
1428 'fuchsia': '#ff00ff',
1429 'gainsboro': '#dcdcdc',
1430 'ghostwhite': '#f8f8ff',
1431 'gold': '#ffd700',
1432 'goldenrod': '#daa520',
1433 'gray': '#808080',
1434 'grey': '#808080',
1435 'green': '#008000',
1436 'greenyellow': '#adff2f',
1437 'honeydew': '#f0fff0',
1438 'hotpink': '#ff69b4',
1439 'indianred ': '#cd5c5c',
1440 'indigo ': '#4b0082',
1441 'ivory': '#fffff0',
1442 'khaki': '#f0e68c',
1443 'lavender': '#e6e6fa',
1444 'lavenderblush': '#fff0f5',
1445 'lawngreen': '#7cfc00',
1446 'lemonchiffon': '#fffacd',
1447 'lightblue': '#add8e6',
1448 'lightcoral': '#f08080',
1449 'lightcyan': '#e0ffff',
1450 'lightgoldenrodyellow': '#fafad2',
1451 'lightgray': '#d3d3d3',
1452 'lightgrey': '#d3d3d3',
1453 'lightgreen': '#90ee90',
1454 'lightpink': '#ffb6c1',
1455 'lightsalmon': '#ffa07a',
1456 'lightseagreen': '#20b2aa',
1457 'lightskyblue': '#87cefa',
1458 'lightslategray': '#778899',
1459 'lightslategrey': '#778899',
1460 'lightsteelblue': '#b0c4de',
1461 'lightyellow': '#ffffe0',
1462 'lime': '#00ff00',
1463 'limegreen': '#32cd32',
1464 'linen': '#faf0e6',
1465 'magenta': '#ff00ff',
1466 'maroon': '#800000',
1467 'mediumaquamarine': '#66cdaa',
1468 'mediumblue': '#0000cd',
1469 'mediumorchid': '#ba55d3',
1470 'mediumpurple': '#9370d8',
1471 'mediumseagreen': '#3cb371',
1472 'mediumslateblue': '#7b68ee',
1473 'mediumspringgreen': '#00fa9a',
1474 'mediumturquoise': '#48d1cc',
1475 'mediumvioletred': '#c71585',
1476 'midnightblue': '#191970',
1477 'mintcream': '#f5fffa',
1478 'mistyrose': '#ffe4e1',
1479 'moccasin': '#ffe4b5',
1480 'navajowhite': '#ffdead',
1481 'navy': '#000080',
1482 'oldlace': '#fdf5e6',
1483 'olive': '#808000',
1484 'olivedrab': '#6b8e23',
1485 'orange': '#ffa500',
1486 'orangered': '#ff4500',
1487 'orchid': '#da70d6',
1488 'palegoldenrod': '#eee8aa',
1489 'palegreen': '#98fb98',
1490 'paleturquoise': '#afeeee',
1491 'palevioletred': '#d87093',
1492 'papayawhip': '#ffefd5',
1493 'peachpuff': '#ffdab9',
1494 'peru': '#cd853f',
1495 'pink': '#ffc0cb',
1496 'plum': '#dda0dd',
1497 'powderblue': '#b0e0e6',
1498 'purple': '#800080',
1499 'red': '#ff0000',
1500 'rosybrown': '#bc8f8f',
1501 'royalblue': '#4169e1',
1502 'saddlebrown': '#8b4513',
1503 'salmon': '#fa8072',
1504 'sandybrown': '#f4a460',
1505 'seagreen': '#2e8b57',
1506 'seashell': '#fff5ee',
1507 'sienna': '#a0522d',
1508 'silver': '#c0c0c0',
1509 'skyblue': '#87ceeb',
1510 'slateblue': '#6a5acd',
1511 'slategray': '#708090',
1512 'slategrey': '#708090',
1513 'snow': '#fffafa',
1514 'springgreen': '#00ff7f',
1515 'steelblue': '#4682b4',
1516 'tan': '#d2b48c',
1517 'teal': '#008080',
1518 'thistle': '#d8bfd8',
1519 'tomato': '#ff6347',
1520 'turquoise': '#40e0d0',
1521 'violet': '#ee82ee',
1522 'wheat': '#f5deb3',
1523 'white': '#ffffff',
1524 'whitesmoke': '#f5f5f5',
1525 'yellow': '#ffff00',
1526 'yellowgreen': '#9acd32',
1527 }
1528
1529 def create_colors(self, hue, num_colors=5, saturation=None, lightness=None):
1530 if saturation is None:
1531 saturation = 0.9
1532 if lightness is None:
1533 lightness = 40
1534 else:
1535 lightness *= 100
1536
1537 import math
1538 saturation -= math.trunc(saturation)
1539
1540 print(hue, saturation)
1541 import colorsys
1542 ''' Create n related colours '''
1543 colors = []
1544 for i in xrange(num_colors):
1545 ix = i * (1.0/num_colors)
1546 _lightness = (lightness + (ix * 40))/100.
1547 if _lightness > 1.0:
1548 _lightness = 1.0
1549 color = colorsys.hls_to_rgb(hue, _lightness, saturation)
1550 hex_color = '#'
1551 for part in color:
1552 hex_color += '%02x' % int(part * 255)
1553 # check and remove any bad values
1554 if not re.match('^\#[0-9a-f]{6}$', hex_color):
1555 hex_color = '#FFFFFF'
1556 colors.append(hex_color)
1557 return colors
1558
1559 def command(self):
1560
1561 hue = None
1562 saturation = None
1563 lightness = None
1564
1565 public = config.get(u'ckan.base_public_folder')
1566 path = os.path.dirname(__file__)
1567 path = os.path.join(path, '..', public, 'base', 'less', 'custom.less')
1568
1569 if self.args:
1570 arg = self.args[0]
1571 rgb = None
1572 if arg == 'clear':
1573 os.remove(path)
1574 print('custom colors removed.')
1575 elif arg.startswith('#'):
1576 color = arg[1:]
1577 if len(color) == 3:
1578 rgb = [int(x, 16) * 16 for x in color]
1579 elif len(color) == 6:
1580 rgb = [int(x, 16) for x in re.findall('..', color)]
1581 else:
1582 print('ERROR: invalid color')
1583 elif arg.lower() in self.color_list:
1584 color = self.color_list[arg.lower()][1:]
1585 rgb = [int(x, 16) for x in re.findall('..', color)]
1586 else:
1587 try:
1588 hue = float(self.args[0])
1589 except ValueError:
1590 print('ERROR argument `%s` not recognised' % arg)
1591 if rgb:
1592 import colorsys
1593 hue, lightness, saturation = colorsys.rgb_to_hls(*rgb)
1594 lightness = lightness / 340
1595 # deal with greys
1596 if not (hue == 0.0 and saturation == 0.0):
1597 saturation = None
1598 else:
1599 import random
1600 hue = random.random()
1601 if hue is not None:
1602 f = open(path, 'w')
1603 colors = self.create_colors(hue, saturation=saturation, lightness=lightness)
1604 for i in xrange(len(self.rules)):
1605 f.write('%s: %s;\n' % (self.rules[i], colors[i]))
1606 print('%s: %s;\n' % (self.rules[i], colors[i]))
1607 f.close
1608 print('Color scheme has been created.')
1609 print('Make sure less is run for changes to take effect.')
1610
1611
1612class TranslationsCommand(CkanCommand):
1613 '''Translation helper functions
1614
1615 trans js - generate the javascript translations
1616 trans mangle - mangle the zh_TW translations for testing
1617 '''
1618
1619 summary = __doc__.split('\n')[0]
1620 usage = __doc__
1621 max_args = 1
1622 min_args = 1
1623
1624 def command(self):
1625 self._load_config()
1626 from ckan.common import config
1627 from ckan.lib.i18n import build_js_translations
1628 ckan_path = os.path.join(os.path.dirname(__file__), '..')
1629 self.i18n_path = config.get('ckan.i18n_directory',
1630 os.path.join(ckan_path, 'i18n'))
1631 command = self.args[0]
1632 if command == 'mangle':
1633 self.mangle_po()
1634 elif command == 'js':
1635 build_js_translations()
1636 else:
1637 print('command not recognised')
1638
1639 def mangle_po(self):
1640 ''' This will mangle the zh_TW translations for translation coverage
1641 testing.
1642
1643 NOTE: This will destroy the current translations fot zh_TW
1644 '''
1645 import polib
1646 pot_path = os.path.join(self.i18n_path, 'ckan.pot')
1647 po = polib.pofile(pot_path)
1648 # we don't want to mangle the following items in strings
1649 # %(...)s %s %0.3f %1$s %2$0.3f [1:...] {...} etc
1650
1651 # sprintf bit after %
1652 spf_reg_ex = "\+?(0|'.)?-?\d*(.\d*)?[\%bcdeufosxX]"
1653
1654 extract_reg_ex = '(\%\([^\)]*\)' + spf_reg_ex + \
1655 '|\[\d*\:[^\]]*\]' + \
1656 '|\{[^\}]*\}' + \
1657 '|<[^>}]*>' + \
1658 '|\%((\d)*\$)?' + spf_reg_ex + ')'
1659
1660 for entry in po:
1661 msg = entry.msgid.encode('utf-8')
1662 matches = re.finditer(extract_reg_ex, msg)
1663 length = len(msg)
1664 position = 0
1665 translation = u''
1666 for match in matches:
1667 translation += '-' * (match.start() - position)
1668 position = match.end()
1669 translation += match.group(0)
1670 translation += '-' * (length - position)
1671 entry.msgstr = translation
1672 out_dir = os.path.join(self.i18n_path, 'zh_TW', 'LC_MESSAGES')
1673 try:
1674 os.makedirs(out_dir)
1675 except OSError:
1676 pass
1677 po.metadata['Plural-Forms'] = "nplurals=1; plural=0\n"
1678 out_po = os.path.join(out_dir, 'ckan.po')
1679 out_mo = os.path.join(out_dir, 'ckan.mo')
1680 po.save(out_po)
1681 po.save_as_mofile(out_mo)
1682 print('zh_TW has been mangled')
1683
1684
1685class MinifyCommand(CkanCommand):
1686 '''Create minified versions of the given Javascript and CSS files.
1687
1688 Usage:
1689
1690 paster minify [--clean] PATH
1691
1692 for example:
1693
1694 paster minify ckan/public/base
1695 paster minify ckan/public/base/css/*.css
1696 paster minify ckan/public/base/css/red.css
1697
1698 if the --clean option is provided any minified files will be removed.
1699
1700 '''
1701 summary = __doc__.split('\n')[0]
1702 usage = __doc__
1703 min_args = 1
1704
1705 exclude_dirs = ['vendor']
1706
1707 def __init__(self, name):
1708
1709 super(MinifyCommand, self).__init__(name)
1710
1711 self.parser.add_option('--clean', dest='clean',
1712 action='store_true', default=False,
1713 help='remove any minified files in the path')
1714
1715 def command(self):
1716 clean = getattr(self.options, 'clean', False)
1717 self._load_config()
1718 for base_path in self.args:
1719 if os.path.isfile(base_path):
1720 if clean:
1721 self.clear_minifyed(base_path)
1722 else:
1723 self.minify_file(base_path)
1724 elif os.path.isdir(base_path):
1725 for root, dirs, files in os.walk(base_path):
1726 dirs[:] = [d for d in dirs if not d in self.exclude_dirs]
1727 for filename in files:
1728 path = os.path.join(root, filename)
1729 if clean:
1730 self.clear_minifyed(path)
1731 else:
1732 self.minify_file(path)
1733 else:
1734 # Path is neither a file or a dir?
1735 continue
1736
1737 def clear_minifyed(self, path):
1738 path_only, extension = os.path.splitext(path)
1739
1740 if extension not in ('.css', '.js'):
1741 # This is not a js or css file.
1742 return
1743
1744 if path_only.endswith('.min'):
1745 print('removing %s' % path)
1746 os.remove(path)
1747
1748 def minify_file(self, path):
1749 '''Create the minified version of the given file.
1750
1751 If the file is not a .js or .css file (e.g. it's a .min.js or .min.css
1752 file, or it's some other type of file entirely) it will not be
1753 minifed.
1754
1755 :param path: The path to the .js or .css file to minify
1756
1757 '''
1758 import ckan.lib.fanstatic_resources as fanstatic_resources
1759
1760 path_only, extension = os.path.splitext(path)
1761
1762 if path_only.endswith('.min'):
1763 # This is already a minified file.
1764 return
1765
1766 if extension not in ('.css', '.js'):
1767 # This is not a js or css file.
1768 return
1769
1770 path_min = fanstatic_resources.min_path(path)
1771
1772 source = open(path, 'r').read()
1773 f = open(path_min, 'w')
1774 if path.endswith('.css'):
1775 f.write(rcssmin.cssmin(source))
1776 elif path.endswith('.js'):
1777 f.write(rjsmin.jsmin(source))
1778 f.close()
1779 print("Minified file '{0}'".format(path))
1780
1781
1782class LessCommand(CkanCommand):
1783 '''Compile all root less documents into their CSS counterparts
1784
1785 Usage:
1786
1787 paster less
1788
1789 '''
1790 summary = __doc__.split('\n')[0]
1791 usage = __doc__
1792 min_args = 0
1793
1794 def command(self):
1795 self._load_config()
1796 self.less()
1797
1798 custom_css = {
1799 'fuchsia': '''
1800 @layoutLinkColor: #E73892;
1801 @footerTextColor: mix(#FFF, @layoutLinkColor, 60%);
1802 @footerLinkColor: @footerTextColor;
1803 @mastheadBackgroundColor: @layoutLinkColor;
1804 @btnPrimaryBackground: lighten(@layoutLinkColor, 10%);
1805 @btnPrimaryBackgroundHighlight: @layoutLinkColor;
1806 ''',
1807
1808 'green': '''
1809 @layoutLinkColor: #2F9B45;
1810 @footerTextColor: mix(#FFF, @layoutLinkColor, 60%);
1811 @footerLinkColor: @footerTextColor;
1812 @mastheadBackgroundColor: @layoutLinkColor;
1813 @btnPrimaryBackground: lighten(@layoutLinkColor, 10%);
1814 @btnPrimaryBackgroundHighlight: @layoutLinkColor;
1815 ''',
1816
1817 'red': '''
1818 @layoutLinkColor: #C14531;
1819 @footerTextColor: mix(#FFF, @layoutLinkColor, 60%);
1820 @footerLinkColor: @footerTextColor;
1821 @mastheadBackgroundColor: @layoutLinkColor;
1822 @btnPrimaryBackground: lighten(@layoutLinkColor, 10%);
1823 @btnPrimaryBackgroundHighlight: @layoutLinkColor;
1824 ''',
1825
1826 'maroon': '''
1827 @layoutLinkColor: #810606;
1828 @footerTextColor: mix(#FFF, @layoutLinkColor, 60%);
1829 @footerLinkColor: @footerTextColor;
1830 @mastheadBackgroundColor: @layoutLinkColor;
1831 @btnPrimaryBackground: lighten(@layoutLinkColor, 10%);
1832 @btnPrimaryBackgroundHighlight: @layoutLinkColor;
1833 ''',
1834 }
1835
1836 def less(self):
1837 ''' Compile less files '''
1838 import subprocess
1839 command = ('npm', 'bin')
1840 process = subprocess.Popen(
1841 command,
1842 stdout=subprocess.PIPE,
1843 stderr=subprocess.PIPE,
1844 universal_newlines=True)
1845 output = process.communicate()
1846 directory = output[0].strip()
1847 if not directory:
1848 raise error('Command "{}" returned nothing. Check that npm is '
1849 'installed.'.format(' '.join(command)))
1850 less_bin = os.path.join(directory, 'lessc')
1851
1852 public = config.get(u'ckan.base_public_folder')
1853
1854 root = os.path.join(os.path.dirname(__file__), '..', public, 'base')
1855 root = os.path.abspath(root)
1856 custom_less = os.path.join(root, 'less', 'custom.less')
1857 for color in self.custom_css:
1858 f = open(custom_less, 'w')
1859 f.write(self.custom_css[color])
1860 f.close()
1861 self.compile_less(root, less_bin, color)
1862 f = open(custom_less, 'w')
1863 f.write('// This file is needed in order for `gulp build` to compile in less 1.3.1+\n')
1864 f.close()
1865 self.compile_less(root, less_bin, 'main')
1866
1867 def compile_less(self, root, less_bin, color):
1868 print('compile %s.css' % color)
1869 import subprocess
1870 main_less = os.path.join(root, 'less', 'main.less')
1871 main_css = os.path.join(root, 'css', '%s.css' % color)
1872
1873 command = (less_bin, main_less, main_css)
1874 process = subprocess.Popen(
1875 command,
1876 stdout=subprocess.PIPE,
1877 stderr=subprocess.PIPE,
1878 universal_newlines=True)
1879 output = process.communicate()
1880 print(output)
1881
1882
1883class FrontEndBuildCommand(CkanCommand):
1884 '''Creates and minifies css and JavaScript files
1885
1886 Usage:
1887
1888 paster front-end-build
1889 '''
1890
1891 summary = __doc__.split('\n')[0]
1892 usage = __doc__
1893 min_args = 0
1894
1895 def command(self):
1896 self._load_config()
1897
1898 # Less css
1899 cmd = LessCommand('less')
1900 cmd.options = self.options
1901 cmd.command()
1902
1903 # js translation strings
1904 cmd = TranslationsCommand('trans')
1905 cmd.options = self.options
1906 cmd.args = ('js',)
1907 cmd.command()
1908
1909 # minification
1910 cmd = MinifyCommand('minify')
1911 cmd.options = self.options
1912 public = config.get(u'ckan.base_public_folder')
1913 root = os.path.join(os.path.dirname(__file__), '..', public, 'base')
1914 root = os.path.abspath(root)
1915 ckanext = os.path.join(os.path.dirname(__file__), '..', '..', 'ckanext')
1916 ckanext = os.path.abspath(ckanext)
1917 cmd.args = (root, ckanext)
1918 cmd.command()
1919
1920
1921class ViewsCommand(CkanCommand):
1922 '''Manage resource views.
1923
1924 Usage:
1925
1926 paster views create [options] [type1] [type2] ...
1927
1928 Create views on relevant resources. You can optionally provide
1929 specific view types (eg `recline_view`, `image_view`). If no types
1930 are provided, the default ones will be used. These are generally
1931 the ones defined in the `ckan.views.default_views` config option.
1932 Note that on either case, plugins must be loaded (ie added to
1933 `ckan.plugins`), otherwise the command will stop.
1934
1935 paster views clear [options] [type1] [type2] ...
1936
1937 Permanently delete all views or the ones with the provided types.
1938
1939 paster views clean
1940
1941 Permanently delete views for all types no longer present in the
1942 `ckan.plugins` configuration option.
1943
1944 '''
1945
1946 summary = __doc__.split('\n')[0]
1947 usage = __doc__
1948 min_args = 1
1949
1950 def __init__(self, name):
1951
1952 super(ViewsCommand, self).__init__(name)
1953
1954 self.parser.add_option('-y', '--yes', dest='assume_yes',
1955 action='store_true',
1956 default=False,
1957 help='''Automatic yes to prompts. Assume "yes"
1958as answer to all prompts and run non-interactively''')
1959
1960 self.parser.add_option('-d', '--dataset', dest='dataset_id',
1961 action='append',
1962 help='''Create views on a particular dataset.
1963You can use the dataset id or name, and it can be defined multiple times.''')
1964
1965 self.parser.add_option('--no-default-filters',
1966 dest='no_default_filters',
1967 action='store_true',
1968 default=False,
1969 help='''Do not add default filters for relevant
1970resource formats for the view types provided. Note that filters are not added
1971by default anyway if an unsupported view type is provided or when using the
1972`-s` or `-d` options.''')
1973
1974 self.parser.add_option('-s', '--search', dest='search_params',
1975 action='store',
1976 default=False,
1977 help='''Extra search parameters that will be
1978used for getting the datasets to create the resource views on. It must be a
1979JSON object like the one used by the `package_search` API call. Supported
1980fields are `q`, `fq` and `fq_list`. Check the documentation for examples.
1981Not used when using the `-d` option.''')
1982
1983 def command(self):
1984 self._load_config()
1985 if not self.args:
1986 print(self.usage)
1987 elif self.args[0] == 'create':
1988 view_plugin_types = self.args[1:]
1989 self.create_views(view_plugin_types)
1990 elif self.args[0] == 'clear':
1991 view_plugin_types = self.args[1:]
1992 self.clear_views(view_plugin_types)
1993 elif self.args[0] == 'clean':
1994 self.clean_views()
1995 else:
1996 print(self.usage)
1997
1998 _page_size = 100
1999
2000 def _get_view_plugins(self, view_plugin_types,
2001 get_datastore_views=False):
2002 '''
2003 Returns the view plugins that were succesfully loaded
2004
2005 Views are provided as a list of ``view_plugin_types``. If no types are
2006 provided, the default views defined in the ``ckan.views.default_views``
2007 will be created. Only in this case (when the default view plugins are
2008 used) the `get_datastore_views` parameter can be used to get also view
2009 plugins that require data to be in the DataStore.
2010
2011 If any of the provided plugins could not be loaded (eg it was not added
2012 to `ckan.plugins`) the command will stop.
2013
2014 Returns a list of loaded plugin names.
2015 '''
2016 from ckan.lib.datapreview import (get_view_plugins,
2017 get_default_view_plugins
2018 )
2019
2020 log = logging.getLogger(__name__)
2021
2022 view_plugins = []
2023
2024 if not view_plugin_types:
2025 log.info('No view types provided, using default types')
2026 view_plugins = get_default_view_plugins()
2027 if get_datastore_views:
2028 view_plugins.extend(
2029 get_default_view_plugins(get_datastore_views=True))
2030 else:
2031 view_plugins = get_view_plugins(view_plugin_types)
2032
2033 loaded_view_plugins = [view_plugin.info()['name']
2034 for view_plugin in view_plugins]
2035
2036 plugins_not_found = list(set(view_plugin_types) -
2037 set(loaded_view_plugins))
2038
2039 if plugins_not_found:
2040 error('View plugin(s) not found : {0}. '.format(plugins_not_found)
2041 + 'Have they been added to the `ckan.plugins` configuration'
2042 + ' option?')
2043
2044 return loaded_view_plugins
2045
2046 def _add_default_filters(self, search_data_dict, view_types):
2047 '''
2048 Adds extra filters to the `package_search` dict for common view types
2049
2050 It basically adds `fq` parameters that filter relevant resource formats
2051 for the view types provided. For instance, if one of the view types is
2052 `pdf_view` the following will be added to the final query:
2053
2054 fq=res_format:"pdf" OR res_format:"PDF"
2055
2056 This obviously should only be used if all view types are known and can
2057 be filtered, otherwise we want all datasets to be returned. If a
2058 non-filterable view type is provided, the search params are not
2059 modified.
2060
2061 Returns the provided data_dict for `package_search`, optionally
2062 modified with extra filters.
2063 '''
2064
2065 from ckanext.imageview.plugin import DEFAULT_IMAGE_FORMATS
2066 from ckanext.textview.plugin import get_formats as get_text_formats
2067 from ckanext.datapusher.plugin import DEFAULT_FORMATS as \
2068 datapusher_formats
2069
2070 filter_formats = []
2071
2072 for view_type in view_types:
2073 if view_type == 'image_view':
2074
2075 for _format in DEFAULT_IMAGE_FORMATS:
2076 filter_formats.extend([_format, _format.upper()])
2077
2078 elif view_type == 'text_view':
2079 formats = get_text_formats(config)
2080 for _format in itertools.chain.from_iterable(formats.values()):
2081 filter_formats.extend([_format, _format.upper()])
2082
2083 elif view_type == 'pdf_view':
2084 filter_formats.extend(['pdf', 'PDF'])
2085
2086 elif view_type in ['recline_view', 'recline_grid_view',
2087 'recline_graph_view', 'recline_map_view']:
2088
2089 if datapusher_formats[0] in filter_formats:
2090 continue
2091
2092 for _format in datapusher_formats:
2093 if '/' not in _format:
2094 filter_formats.extend([_format, _format.upper()])
2095 else:
2096 # There is another view type provided so we can't add any
2097 # filter
2098 return search_data_dict
2099
2100 filter_formats_query = ['+res_format:"{0}"'.format(_format)
2101 for _format in filter_formats]
2102 search_data_dict['fq_list'].append(' OR '.join(filter_formats_query))
2103
2104 return search_data_dict
2105
2106 def _update_search_params(self, search_data_dict):
2107 '''
2108 Update the `package_search` data dict with the user provided parameters
2109
2110 Supported fields are `q`, `fq` and `fq_list`.
2111
2112 If the provided JSON object can not be parsed the process stops with
2113 an error.
2114
2115 Returns the updated data dict
2116 '''
2117
2118 log = logging.getLogger(__name__)
2119
2120 if not self.options.search_params:
2121 return search_data_dict
2122
2123 try:
2124 user_search_params = json.loads(self.options.search_params)
2125 except ValueError as e:
2126 error('Unable to parse JSON search parameters: {0}'.format(e))
2127
2128 if user_search_params.get('q'):
2129 search_data_dict['q'] = user_search_params['q']
2130
2131 if user_search_params.get('fq'):
2132 if search_data_dict['fq']:
2133 search_data_dict['fq'] += ' ' + user_search_params['fq']
2134 else:
2135 search_data_dict['fq'] = user_search_params['fq']
2136
2137 if (user_search_params.get('fq_list') and
2138 isinstance(user_search_params['fq_list'], list)):
2139 search_data_dict['fq_list'].extend(user_search_params['fq_list'])
2140
2141 def _search_datasets(self, page=1, view_types=[]):
2142 '''
2143 Perform a query with `package_search` and return the result
2144
2145 Results can be paginated using the `page` parameter
2146 '''
2147
2148 n = self._page_size
2149
2150 search_data_dict = {
2151 'q': '',
2152 'fq': '',
2153 'fq_list': [],
2154 'include_private': True,
2155 'rows': n,
2156 'start': n * (page - 1),
2157 }
2158
2159 if self.options.dataset_id:
2160
2161 search_data_dict['q'] = ' OR '.join(
2162 ['id:{0} OR name:"{0}"'.format(dataset_id)
2163 for dataset_id in self.options.dataset_id]
2164 )
2165
2166 elif self.options.search_params:
2167
2168 self._update_search_params(search_data_dict)
2169
2170 elif not self.options.no_default_filters:
2171
2172 self._add_default_filters(search_data_dict, view_types)
2173
2174 if not search_data_dict.get('q'):
2175 search_data_dict['q'] = '*:*'
2176
2177 query = p.toolkit.get_action('package_search')(
2178 {}, search_data_dict)
2179
2180 return query
2181
2182 def create_views(self, view_plugin_types=[]):
2183
2184 from ckan.lib.datapreview import add_views_to_dataset_resources
2185
2186 log = logging.getLogger(__name__)
2187
2188 datastore_enabled = 'datastore' in config['ckan.plugins'].split()
2189
2190 loaded_view_plugins = self._get_view_plugins(view_plugin_types,
2191 datastore_enabled)
2192
2193 context = {'user': self.site_user['name']}
2194
2195 page = 1
2196 while True:
2197 query = self._search_datasets(page, loaded_view_plugins)
2198
2199 if page == 1 and query['count'] == 0:
2200 error('No datasets to create resource views on, exiting...')
2201
2202 elif page == 1 and not self.options.assume_yes:
2203
2204 msg = ('\nYou are about to check {0} datasets for the ' +
2205 'following view plugins: {1}\n' +
2206 ' Do you want to continue?')
2207
2208 confirm = query_yes_no(msg.format(query['count'],
2209 loaded_view_plugins))
2210
2211 if confirm == 'no':
2212 error('Command aborted by user')
2213
2214 if query['results']:
2215 for dataset_dict in query['results']:
2216
2217 if not dataset_dict.get('resources'):
2218 continue
2219
2220 views = add_views_to_dataset_resources(
2221 context,
2222 dataset_dict,
2223 view_types=loaded_view_plugins)
2224
2225 if views:
2226 view_types = list({view['view_type']
2227 for view in views})
2228 msg = ('Added {0} view(s) of type(s) {1} to ' +
2229 'resources from dataset {2}')
2230 log.debug(msg.format(len(views),
2231 ', '.join(view_types),
2232 dataset_dict['name']))
2233
2234 if len(query['results']) < self._page_size:
2235 break
2236
2237 page += 1
2238 else:
2239 break
2240
2241 log.info('Done')
2242
2243 def clear_views(self, view_plugin_types=[]):
2244
2245 log = logging.getLogger(__name__)
2246
2247 if not self.options.assume_yes:
2248 if view_plugin_types:
2249 msg = 'Are you sure you want to delete all resource views ' + \
2250 'of type {0}?'.format(', '.join(view_plugin_types))
2251 else:
2252 msg = 'Are you sure you want to delete all resource views?'
2253
2254 result = query_yes_no(msg, default='no')
2255
2256 if result == 'no':
2257 error('Command aborted by user')
2258
2259 context = {'user': self.site_user['name']}
2260 logic.get_action('resource_view_clear')(
2261 context, {'view_types': view_plugin_types})
2262
2263 log.info('Done')
2264
2265 def clean_views(self):
2266 names = []
2267 for plugin in p.PluginImplementations(p.IResourceView):
2268 names.append(str(plugin.info()['name']))
2269
2270 results = model.ResourceView.get_count_not_in_view_types(names)
2271
2272 if not results:
2273 print('No resource views to delete')
2274 return
2275
2276 print('This command will delete.\n')
2277 for row in results:
2278 print('%s of type %s' % (row[1], row[0]))
2279
2280 result = query_yes_no('Do you want to delete these resource views:', default='no')
2281
2282 if result == 'no':
2283 print('Not Deleting.')
2284 return
2285
2286 model.ResourceView.delete_not_in_view_types(names)
2287 model.Session.commit()
2288 print('Deleted resource views.')
2289
2290
2291class ConfigToolCommand(paste.script.command.Command):
2292 '''Tool for editing options in a CKAN config file
2293
2294 paster config-tool <default.ini> <key>=<value> [<key>=<value> ...]
2295 paster config-tool <default.ini> -f <custom_options.ini>
2296
2297 Examples:
2298 paster config-tool default.ini sqlalchemy.url=123 'ckan.site_title=ABC'
2299 paster config-tool default.ini -s server:main -e port=8080
2300 paster config-tool default.ini -f custom_options.ini
2301 '''
2302 parser = paste.script.command.Command.standard_parser(verbose=True)
2303 default_verbosity = 1
2304 group_name = 'ckan'
2305 usage = __doc__
2306 summary = usage.split('\n')[0]
2307
2308 parser.add_option('-s', '--section', dest='section',
2309 default='app:main', help='Section of the config file')
2310 parser.add_option(
2311 '-e', '--edit', action='store_true', dest='edit', default=False,
2312 help='Checks the option already exists in the config file')
2313 parser.add_option(
2314 '-f', '--file', dest='merge_filepath', metavar='FILE',
2315 help='Supply an options file to merge in')
2316
2317 def command(self):
2318 from ckan.lib import config_tool
2319 if len(self.args) < 1:
2320 self.parser.error('Not enough arguments (got %i, need at least 1)'
2321 % len(self.args))
2322 config_filepath = self.args[0]
2323 if not os.path.exists(config_filepath):
2324 self.parser.error('Config filename %r does not exist.' %
2325 config_filepath)
2326 if self.options.merge_filepath:
2327 config_tool.config_edit_using_merge_file(
2328 config_filepath, self.options.merge_filepath)
2329 options = self.args[1:]
2330 if not (options or self.options.merge_filepath):
2331 self.parser.error('No options provided')
2332 if options:
2333 for option in options:
2334 if '=' not in option:
2335 error(
2336 'An option does not have an equals sign: %r '
2337 'It should be \'key=value\'. If there are spaces '
2338 'you\'ll need to quote the option.\n' % option)
2339 try:
2340 config_tool.config_edit_using_option_strings(
2341 config_filepath, options, self.options.section,
2342 edit=self.options.edit)
2343 except config_tool.ConfigToolError as e:
2344 error(traceback.format_exc())
2345
2346
2347class JobsCommand(CkanCommand):
2348 '''Manage background jobs
2349
2350 Usage:
2351
2352 paster jobs worker [--burst] [QUEUES]
2353
2354 Start a worker that fetches jobs from queues and executes
2355 them. If no queue names are given then the worker listens
2356 to the default queue, this is equivalent to
2357
2358 paster jobs worker default
2359
2360 If queue names are given then the worker listens to those
2361 queues and only those:
2362
2363 paster jobs worker my-custom-queue
2364
2365 Hence, if you want the worker to listen to the default queue
2366 and some others then you must list the default queue explicitly:
2367
2368 paster jobs worker default my-custom-queue
2369
2370 If the `--burst` option is given then the worker will exit
2371 as soon as all its queues are empty.
2372
2373 paster jobs list [QUEUES]
2374
2375 List currently enqueued jobs from the given queues. If no queue
2376 names are given then the jobs from all queues are listed.
2377
2378 paster jobs show ID
2379
2380 Show details about a specific job.
2381
2382 paster jobs cancel ID
2383
2384 Cancel a specific job. Jobs can only be canceled while they are
2385 enqueued. Once a worker has started executing a job it cannot
2386 be aborted anymore.
2387
2388 paster jobs clear [QUEUES]
2389
2390 Cancel all jobs on the given queues. If no queue names are
2391 given then ALL queues are cleared.
2392
2393 paster jobs test [QUEUES]
2394
2395 Enqueue a test job. If no queue names are given then the job is
2396 added to the default queue. If queue names are given then a
2397 separate test job is added to each of the queues.
2398 '''
2399
2400 summary = __doc__.split(u'\n')[0]
2401 usage = __doc__
2402 min_args = 0
2403
2404
2405 def __init__(self, *args, **kwargs):
2406 super(JobsCommand, self).__init__(*args, **kwargs)
2407 try:
2408 self.parser.add_option(u'--burst', action='store_true',
2409 default=False,
2410 help=u'Start worker in burst mode.')
2411 except OptionConflictError:
2412 # Option has already been added in previous call
2413 pass
2414
2415 def command(self):
2416 self._load_config()
2417 try:
2418 cmd = self.args.pop(0)
2419 except IndexError:
2420 print(self.__doc__)
2421 sys.exit(0)
2422 if cmd == u'worker':
2423 self.worker()
2424 elif cmd == u'list':
2425 self.list()
2426 elif cmd == u'show':
2427 self.show()
2428 elif cmd == u'cancel':
2429 self.cancel()
2430 elif cmd == u'clear':
2431 self.clear()
2432 elif cmd == u'test':
2433 self.test()
2434 else:
2435 error(u'Unknown command "{}"'.format(cmd))
2436
2437 def worker(self):
2438 from ckan.lib.jobs import Worker
2439 Worker(self.args).work(burst=self.options.burst)
2440
2441 def list(self):
2442 data_dict = {
2443 u'queues': self.args,
2444 }
2445 jobs = p.toolkit.get_action(u'job_list')({}, data_dict)
2446 for job in jobs:
2447 if job[u'title'] is None:
2448 job[u'title'] = ''
2449 else:
2450 job[u'title'] = u'"{}"'.format(job[u'title'])
2451 print(u'{created} {id} {queue} {title}'.format(**job))
2452
2453 def show(self):
2454 if not self.args:
2455 error(u'You must specify a job ID')
2456 id = self.args[0]
2457 try:
2458 job = p.toolkit.get_action(u'job_show')({}, {u'id': id})
2459 except logic.NotFound:
2460 error(u'There is no job with ID "{}"'.format(id))
2461 print(u'ID: {}'.format(job[u'id']))
2462 if job[u'title'] is None:
2463 title = u'None'
2464 else:
2465 title = u'"{}"'.format(job[u'title'])
2466 print(u'Title: {}'.format(title))
2467 print(u'Created: {}'.format(job[u'created']))
2468 print(u'Queue: {}'.format(job[u'queue']))
2469
2470 def cancel(self):
2471 if not self.args:
2472 error(u'You must specify a job ID')
2473 id = self.args[0]
2474 try:
2475 p.toolkit.get_action(u'job_cancel')({}, {u'id': id})
2476 except logic.NotFound:
2477 error(u'There is no job with ID "{}"'.format(id))
2478 print(u'Cancelled job {}'.format(id))
2479
2480 def clear(self):
2481 data_dict = {
2482 u'queues': self.args,
2483 }
2484 queues = p.toolkit.get_action(u'job_clear')({}, data_dict)
2485 queues = (u'"{}"'.format(q) for q in queues)
2486 print(u'Cleared queue(s) {}'.format(u', '.join(queues)))
2487
2488 def test(self):
2489 from ckan.lib.jobs import DEFAULT_QUEUE_NAME, enqueue, test_job
2490 for queue in (self.args or [DEFAULT_QUEUE_NAME]):
2491 job = enqueue(test_job, [u'A test job'], title=u'A test job', queue=queue)
2492 print(u'Added test job {} to queue "{}"'.format(job.id, queue))