· 6 years ago · Feb 19, 2020, 01:56 AM
1""" The portalpy module for working with the ArcGIS Online and Portal APIs."""
2
3__version__ = '1.0'
4
5import collections
6import copy
7import gzip
8import httplib
9import imghdr
10import json
11import logging
12import mimetools
13import mimetypes
14import os
15import re
16import tempfile
17import unicodedata
18import urllib
19import urllib2
20import urlparse
21from cStringIO import StringIO
22
23
24
25_log = logging.getLogger(__name__)
26
27class Portal(object):
28 """ An object representing a connection to a single portal (via URL).
29
30 .. note:: To instantiate a Portal object execute code like this:
31
32 PortalPy.Portal(portalUrl, user, password)
33
34 There are a few things you should know as you use the methods below.
35
36 Group IDs - Many of the group functions require a group id. This id is
37 different than the group's name or title. To determine
38 a group id, use the search_groups function using the title
39 to get the group id.
40
41 Time - Many of the methods return a time field. All time is
42 returned as millseconds since 1 January 1970. Python
43 expects time in seconds since 1 January 1970 so make sure
44 to divide times from PortalPy by 1000. See the example
45 a few lines down to see how to convert from PortalPy time
46 to Python time.
47
48 Example - converting time
49
50 .. code-block:: python
51
52 import time
53 .
54 .
55 .
56 group = portalAdmin.get_group('67e1761068b7453693a0c68c92a62e2e')
57 pythontime = time.ctime(group['created']/1000)
58
59 Example - list users in group
60
61 .. code-block:: python
62
63 portal = PortalPy.Portal(portalUrl, user, password)
64 resp = portal.get_group_members('67e1761068b7453693a0c68c92a62e2e')
65 for user in resp['users']:
66 print user
67
68 Example - create a group
69
70 .. code-block:: python
71
72 portal= PortalPy.Portal(portalUrl, user, password)
73 group_id = portalAdmin.create_group('my group', 'test tag', 'a group to share travel maps')
74
75 Example - delete a user named amy and assign her content to bob
76
77 .. code-block:: python
78
79 portal = PortalPy.Portal(portalUrl, user, password)
80 portal.delete_user('amy.user', True, 'bob.user')
81
82 """
83
84
85 def __init__(self, url, username=None, password=None, key_file=None,
86 cert_file=None, expiration=60, referer=None, proxy_host=None,
87 proxy_port=None, connection=None, workdir=tempfile.gettempdir()):
88 """ The Portal constructor. Requires URL and optionally username/password."""
89
90 self.url = url
91 if url:
92 normalized_url = _normalize_url(self.url)
93 if not normalized_url[-1] == '/':
94 normalized_url += '/'
95 self.resturl = normalized_url + 'sharing/rest/'
96 self.hostname = _parse_hostname(url)
97 self.workdir = workdir
98
99 # Setup the instance members
100 self._basepostdata = { 'f': 'json' }
101 self._version = None
102 self._properties = None
103 self._logged_in_user = None
104 self._resources = None
105 self._languages = None
106 self._regions = None
107 self._is_pre_162 = False
108 self._is_pre_21 = False
109
110 # If a connection was passed in, use it, otherwise setup the
111 # connection (use all SSL until portal informs us otherwise)
112 if connection:
113 _log.debug('Using existing connection to: ' + \
114 _parse_hostname(connection.baseurl))
115 self.con = connection
116 if not connection:
117 _log.debug('Connecting to portal: ' + self.hostname)
118
119 self.con = _ArcGISConnection(self.resturl, username, password,
120 key_file, cert_file, expiration, True,
121 referer, proxy_host, proxy_port)
122
123 # Store the logged in user information. It's useful.
124 if self.is_logged_in():
125 self._logged_in_user = self.get_user(username)
126
127 self.get_version(True)
128 self.get_properties(True)
129
130
131
132 def add_group_users(self, user_names, group_id):
133 """ Adds users to the group specified.
134
135 .. note::
136 This method will only work if the user for the
137 Portal object is either an administrator for the entire
138 Portal or the owner of the group.
139
140 ============ ======================================
141 **Argument** **Description**
142 ------------ --------------------------------------
143 user_names required string, comma-separated users
144 ------------ --------------------------------------
145 group_id required string, specifying group id
146 ============ ======================================
147
148 :return:
149 A dictionary with a key of "not_added" which contains the users that were not
150 added to the group.
151 """
152
153
154 if self._is_pre_21:
155 _log.warning('The auto_accept option is not supported in ' \
156 + 'pre-2.0 portals')
157 return
158
159 user_names = _unpack(user_names, 'username')
160
161 postdata = self._postdata()
162 postdata['users'] = ','.join(user_names)
163 resp = self.con.post('community/groups/' + group_id + '/addUsers',
164 postdata)
165 return resp
166
167
168 def add_item(self, item_properties, data=None, thumbnail=None, metadata=None, owner=None, folder=None):
169 """ Adds content to a Portal.
170
171
172 .. note::
173 That content can be a file (such as a layer package, geoprocessing package,
174 map package) or it can be a URL (to an ArcGIS Server service, WMS service,
175 or an application).
176
177 If you are uploading a package or other file, provide a path or URL
178 to the file in the data argument.
179
180 From a technical perspective, none of the item properties below are required. However,
181 it is strongly recommended that title, type, typeKeywords, tags, snippet, and description
182 be provided.
183
184
185 ============ ====================================================
186 **Argument** **Description**
187 ------------ ----------------------------------------------------
188 item_properties required dictionary, see below for the keys and values
189 ------------ ----------------------------------------------------
190 data optional string, either a path or URL to the data
191 ------------ ----------------------------------------------------
192 thumbnail optional string, either a path or URL to an image
193 ------------ ----------------------------------------------------
194 metadata optional string, either a path or URL to metadata.
195 ------------ ----------------------------------------------------
196 owner optional string, defaults to logged in user.
197 ------------ ----------------------------------------------------
198 folder optional string, content folder where placing item
199 ============ ====================================================
200
201
202 ================ ============================================================================
203 **Key** **Value**
204 ---------------- ----------------------------------------------------------------------------
205 type optional string, indicates type of item. See URL 1 below for valid values.
206 ---------------- ----------------------------------------------------------------------------
207 typeKeywords optinal string list. Lists all sub-types. See URL 1 for valid values.
208 ---------------- ----------------------------------------------------------------------------
209 description optional string. Description of the item.
210 ---------------- ----------------------------------------------------------------------------
211 title optional string. Name of the item.
212 ---------------- ----------------------------------------------------------------------------
213 url optional string. URL to item that are based on URLs.
214 ---------------- ----------------------------------------------------------------------------
215 tags optional string of comma-separated values. Used for searches on items.
216 ---------------- ----------------------------------------------------------------------------
217 snippet optional string. Provides a very short summary of the what the item is.
218 ---------------- ----------------------------------------------------------------------------
219 extent optional string with comma separated values for min x, min y, max x, max y.
220 ---------------- ----------------------------------------------------------------------------
221 spatialReference optional string. Coordinate system that the item is in.
222 ---------------- ----------------------------------------------------------------------------
223 accessInformation optional string. Information on the source of the content.
224 ---------------- ----------------------------------------------------------------------------
225 licenseInfo optinal string, any license information or restrictions regarding the content.
226 ---------------- ----------------------------------------------------------------------------
227 culture optional string. Locale, country and language information.
228 ---------------- ----------------------------------------------------------------------------
229 access optional string. Valid values: private, shared, org, or public.
230 ---------------- ----------------------------------------------------------------------------
231 commentsEnabled optional boolean. Default is true. Controls whether comments are allowed.
232 ---------------- ----------------------------------------------------------------------------
233 culture optional string. Language and country information.
234 ================ ============================================================================
235
236
237 URL 1: http://resources.arcgis.com/en/help/arcgis-rest-api/index.html#//02r3000000ms000000
238
239 :return:
240 The item id of the uploaded item if successful, None if unsuccessful.
241 """
242
243
244 # Postdata is a dictionary object whose keys and values will be sent via an HTTP Post.
245 postdata = self._postdata()
246 postdata.update(_unicode_to_ascii(item_properties))
247
248 # Build the files list (tuples)
249 files = []
250 if data:
251 if _is_http_url(data):
252 data = urllib.urlretrieve(data)[0]
253 files.append(('file', data, os.path.basename(data)))
254 if metadata:
255 if _is_http_url(metadata):
256 metadata = urllib.urlretrieve(metadata)[0]
257 files.append(('metadata', metadata, 'metadata.xml'))
258 if thumbnail:
259 if _is_http_url(thumbnail):
260 thumbnail = urllib.urlretrieve(thumbnail)[0]
261 file_ext = os.path.splitext(thumbnail)[1]
262 if not file_ext:
263 file_ext = imghdr.what(thumbnail)
264 if file_ext in ('gif', 'png', 'jpeg'):
265 new_thumbnail = thumbnail + '.' + file_ext
266 os.rename(thumbnail, new_thumbnail)
267 thumbnail = new_thumbnail
268 files.append(('thumbnail', thumbnail, os.path.basename(thumbnail)))
269
270 # If owner isn't specified, use the logged in user
271 if not owner:
272 owner = self.logged_in_user()['username']
273
274 # Setup the item path, including the folder, and post to it
275 path = 'content/users/' + owner
276 if folder:
277 path += '/' + folder
278 path += '/addItem'
279 resp = self.con.post(path, postdata, files)
280 if resp and resp.get('success'):
281 return resp['id']
282
283
284
285
286
287 def create_group_from_dict(self, group, thumbnail=None):
288
289 """ Creates a group and returns a group id if successful.
290
291 .. note::
292 Use create_group in most cases. This method is useful for taking a group
293 dict returned from another PortalPy call and copying it.
294
295 ============ ======================================
296 **Argument** **Description**
297 ------------ --------------------------------------
298 group dict object
299 ------------ --------------------------------------
300 thumbnail url to image
301 ============ ======================================
302
303 Example
304
305 .. code-block:: python
306
307 create_group({'title': 'Test', 'access':'public'})
308 """
309
310 postdata = self._postdata()
311 postdata.update(_unicode_to_ascii(group))
312
313 # Build the files list (tuples)
314 files = []
315 if thumbnail:
316 if _is_http_url(thumbnail):
317 thumbnail = urllib.urlretrieve(thumbnail)[0]
318 file_ext = os.path.splitext(thumbnail)[1]
319 if not file_ext:
320 file_ext = imghdr.what(thumbnail)
321 if file_ext in ('gif', 'png', 'jpeg'):
322 new_thumbnail = thumbnail + '.' + file_ext
323 os.rename(thumbnail, new_thumbnail)
324 thumbnail = new_thumbnail
325 files.append(('thumbnail', thumbnail, os.path.basename(thumbnail)))
326
327 # Send the POST request, and return the id from the response
328 resp = self.con.post('community/createGroup', postdata, files)
329 if resp and resp.get('success'):
330 return resp['group']['id']
331
332 def create_group(self, title, tags, description=None,
333 snippet=None, access='public', thumbnail=None,
334 is_invitation_only=False, sort_field='avgRating',
335 sort_order='desc', is_view_only=False, ):
336 """ Creates a group and returns a group id if successful.
337
338 ================ ========================================================
339 **Argument** **Description**
340 ---------------- --------------------------------------------------------
341 title required string, name of the group
342 ---------------- --------------------------------------------------------
343 tags required string, comma-delimited list of tags
344 ---------------- --------------------------------------------------------
345 description optional string, describes group in detail
346 ---------------- --------------------------------------------------------
347 snippet optional string, <250 characters summarizes group
348 ---------------- --------------------------------------------------------
349 access optional string, can be private, public, or org
350 ---------------- --------------------------------------------------------
351 thumbnail optional string, URL to group image
352 ---------------- --------------------------------------------------------
353 isInvitationOnly optional boolean, defines whether users can join by request.
354 ---------------- --------------------------------------------------------
355 sort_field optional string, specifies how shared items with the group are sorted.
356 ---------------- --------------------------------------------------------
357 sort_order optional string, asc or desc for ascending or descending.
358 ---------------- --------------------------------------------------------
359 is_view_only optional boolean, defines whether the group is searchable
360 ================ ========================================================
361
362 :return:
363 a string that is a group id.
364 """
365
366 return self.create_group_from_dict({'title' : title, 'tags' : tags,
367 'snippet' : snippet, 'access' : access,
368 'sortField' : sort_field, 'sortOrder' : sort_order,
369 'isViewOnly' : is_view_only,
370 'isinvitationOnly' : is_invitation_only}, thumbnail)
371
372
373
374
375
376 def delete_group(self, group_id):
377 """ Deletes a group.
378
379 ================ ========================================================
380 **Argument** **Description**
381 ---------------- --------------------------------------------------------
382 group_id string containing the id for the group to be deleted.
383 ================ ========================================================
384
385 Returns
386 a boolean indicating whether it was successful.
387
388 """
389 resp = self.con.post('community/groups/' + group_id + '/delete',
390 self._postdata())
391 if resp:
392 return resp.get('success')
393
394
395 def delete_item(self, item_id, folder=None, owner=None):
396 """ Deletes a single item from Portal.
397
398
399 .. note::
400 The delete item method requires the user to be logged in. Administrators
401 can delete any item in the Portal, but everyone else can only delete
402 their own items.
403
404 When called by an administrator on another user's items, the owner
405 of the item should be specified as an argument.
406
407 The folder in which the item resides must always be provided unless the
408 item is in the user's root folder. If it's in the root folder then the
409 folder argument can be omitted.
410
411 ================ ========================================================
412 **Argument** **Description**
413 ---------------- --------------------------------------------------------
414 item_id required string containing the id of the item to be deleted.
415 ---------------- --------------------------------------------------------
416 owner optional string, the owner of the item, defaults to the logged in user.
417 ---------------- --------------------------------------------------------
418 folder optonal string, the folder in which the item exists. Set to None for root.
419 ================ ========================================================
420
421 Returns
422 a boolean indicating whether it was successful.
423
424 """
425 if owner is None:
426 owner = self.con._username
427
428 if folder is None :
429 path = 'content/users/' + owner + '/items/' + item_id + '/delete'
430 else :
431 path = 'content/users/' + owner + '/' + folder + '/items/' + item_id + '/delete'
432
433 resp = self.con.post(path, self._postdata())
434 if resp:
435 return resp.get('success')
436
437
438
439
440 def delete_items(self, item_ids):
441 """ Deletes multiple items in Portal.
442
443
444 .. note::
445 The delete items method requires the user to be logged in. Administrators
446 can delete any item in the Portal, but everyone else can only delete
447 their own items.
448
449 This method takes a list of item ids.
450
451
452 ================ ========================================================
453 **Argument** **Description**
454 ---------------- --------------------------------------------------------
455 item_ids required list of strings containing the item ids to delete
456 ---------------- --------------------------------------------------------
457 owner optional string, the owner of the item, defaults to the logged in user.
458 ---------------- --------------------------------------------------------
459 folder optonal string, the folder in which the item exists. Set to None for root.
460 ================ ========================================================
461
462 Returns
463 a list of dictionary objects that have itemId and success as the properties.
464
465
466 Example:
467
468 resp = portal.delete_items([item1, item2, item3])
469 for item in resp :
470 print item['itemId'] + ':' + str(item['success'])
471
472 """
473
474 postdata = self._postdata()
475 postdata['items'] = ','.join(item_ids)
476 resp = self.con.post('content/users/' + self.con._username + '/deleteItems', postdata)
477 return resp['results']
478
479
480
481
482 def delete_user(self, username, reassign_to=None):
483 """ Deletes a user from the portal, optionally deleting or reassigning groups and items.
484
485 .. note::
486 You can not delete a user in Portal if that user owns groups or items. If you
487 specify someone in the reassign_to argument then items and groups will be
488 transferred to that user. If that argument is not set then the method
489 will fail if the user has items or groups that need to be reassigned.
490
491
492 ================ ========================================================
493 **Argument** **Description**
494 ---------------- --------------------------------------------------------
495 username required string, the name of the user
496 ---------------- --------------------------------------------------------
497 reassign_to optional string, new owner of items and groups
498 ================ ========================================================
499
500 :return:
501 a boolean indicating whether the operation succeeded or failed.
502
503 """
504
505
506 if reassign_to :
507 self.reassign_user(username, reassign_to)
508 resp = self.con.post('community/users/' + username + '/delete',self._postdata())
509 if resp:
510 return resp.get('success')
511 else:
512 return False
513
514 def generate_token(self, username, password, expiration=60):
515 """ Generates and returns a new token, but doesn't re-login.
516
517 .. note::
518 This method is not needed when using the Portal class
519 to make calls into Portal. It's provided for the benefit
520 of making calls into Portal outside of the Portal class.
521
522 Portal uses a token-based authentication mechanism where
523 a user provides their credentials and a short-term token
524 is used for calls. Most calls made to the Portal REST API
525 require a token and this can be appended to those requests.
526
527 ================ ========================================================
528 **Argument** **Description**
529 ---------------- --------------------------------------------------------
530 username required string, name of the user
531 ---------------- --------------------------------------------------------
532 password required password, name of the user
533 ---------------- --------------------------------------------------------
534 expiration optional integer, number of minutes until the token expires
535 ================ ========================================================
536
537 :return:
538 a string with the token
539
540 """
541
542 return self.con.generate_token(username, password, expiration)
543
544
545 def get_group(self, group_id):
546 """ Returns group information for the specified group group_id.
547
548 Arguments
549 group_id : required string, indicating group.
550
551 :return:
552 a dictionary object with the group's information. The keys in
553 the dictionary object will often include:
554
555 ================ ========================================================
556 **Key** **Value**
557 ---------------- --------------------------------------------------------
558 title: the name of the group
559 ---------------- --------------------------------------------------------
560 isInvitationOnly if set to true, users can't apply to join the group.
561 ---------------- --------------------------------------------------------
562 owner: the owner username of the group
563 ---------------- --------------------------------------------------------
564 description: explains the group
565 ---------------- --------------------------------------------------------
566 snippet: a short summary of the group
567 ---------------- --------------------------------------------------------
568 tags: user-defined tags that describe the group
569 ---------------- --------------------------------------------------------
570 phone: contact information for group.
571 ---------------- --------------------------------------------------------
572 thumbnail: File name relative to http://<community-url>/groups/<groupId>/info
573 ---------------- --------------------------------------------------------
574 created: When group created, ms since 1 Jan 1970
575 ---------------- --------------------------------------------------------
576 modified: When group last modified. ms since 1 Jan 1970
577 ---------------- --------------------------------------------------------
578 access: Can be private, org, or public.
579 ---------------- --------------------------------------------------------
580 userMembership: A dict with keys username and memberType.
581 ---------------- --------------------------------------------------------
582 memberType: provides the calling user's access (owner, admin, member, none).
583 ================ ========================================================
584
585 """
586 return self.con.post('community/groups/' + group_id, self._postdata())
587
588
589
590 def get_group_thumbnail(self, group_id):
591 """ Returns the bytes that make up the thumbnail for the specified group group_id.
592
593 Arguments
594 group_id: required string, specifies the group's thumbnail
595
596 Returns
597 bytes that representt he image.
598
599 Example
600
601 .. code-block:: python
602
603 response = portal.get_group_thumbnail("67e1761068b7453693a0c68c92a62e2e")
604 f = open(filename, 'wb')
605 f.write(response)
606
607 """
608 thumbnail_file = self.get_group(group_id).get('thumbnail')
609 if thumbnail_file:
610 thumbnail_url_path = 'community/groups/' + group_id + '/info/' + thumbnail_file
611 if thumbnail_url_path:
612 return self.con.get(thumbnail_url_path, try_json=False)
613
614
615 def get_group_members(self, group_id):
616 """ Returns members of the specified group.
617
618 Arguments
619 group_id: required string, specifies the group
620
621 Returns
622 a dictionary with keys: owner, admins, and users.
623
624 ================ ========================================================
625 **Key** **Value**
626 ---------------- --------------------------------------------------------
627 owner string value, the group's owner
628 ---------------- --------------------------------------------------------
629 admins list of strings, typically this is the same as the owner.
630 ---------------- --------------------------------------------------------
631 users list of strings, the members of the group
632 ================ ========================================================
633
634 Example (to print users in a group)
635
636 .. code-block:: python
637
638 response = portal.get_group_members("67e1761068b7453693a0c68c92a62e2e")
639 for user in response['users'] :
640 print user
641
642 """
643
644 return self.con.post('community/groups/' + group_id + '/users',
645 self._postdata())
646
647
648 def get_org_users(self, max_users=1000):
649 """ Returns all users within the portal organization.
650
651 Arguments
652 max_users : optional int, the maximum number of users to return.
653
654 :return:
655 a list of dicts. Each dict has the following keys:
656
657 ================ ========================================================
658 **Key** **Value**
659 ---------------- --------------------------------------------------------
660 username : string
661 ---------------- --------------------------------------------------------
662 storageUsage: int
663 ---------------- --------------------------------------------------------
664 storageQuota: int
665 ---------------- --------------------------------------------------------
666 description: string
667 ---------------- --------------------------------------------------------
668 tags: list of strings
669 ---------------- --------------------------------------------------------
670 region: string
671 ---------------- --------------------------------------------------------
672 created: int, when account created, ms since 1 Jan 1970
673 ---------------- --------------------------------------------------------
674 modified: int, when account last modified, ms since 1 Jan 1970
675 ---------------- --------------------------------------------------------
676 email: string
677 ---------------- --------------------------------------------------------
678 culture: string
679 ---------------- --------------------------------------------------------
680 orgId: string
681 ---------------- --------------------------------------------------------
682 preferredView: string
683 ---------------- --------------------------------------------------------
684 groups: list of strings
685 ---------------- --------------------------------------------------------
686 role: string (org_user, org_publisher, org_admin)
687 ---------------- --------------------------------------------------------
688 fullName: string
689 ---------------- --------------------------------------------------------
690 thumbnail: string
691 ---------------- --------------------------------------------------------
692 idpUsername: string
693 ================ ========================================================
694
695 Example (print all usernames in portal):
696
697 .. code-block:: python
698
699 resp = portalAdmin.get_org_users()
700 for user in resp:
701 print user['username']
702
703 """
704
705 # Execute the search and get back the results
706 count = 0
707 resp = self._org_users_page(1, min(max_users, 100))
708 resp_users = resp.get('users')
709 results = resp_users
710 count += int(resp['num'])
711 nextstart = int(resp['nextStart'])
712 while count < max_users and nextstart > 0:
713 resp = self._org_users_page(nextstart, min(max_users - count, 100))
714 resp_users = resp.get('users')
715 results.extend(resp_users)
716 count += int(resp['num'])
717 nextstart = int(resp['nextStart'])
718
719 return results
720
721
722
723 def get_properties(self, force=False):
724 """ Returns the portal properties (using cache unless force=True). """
725
726 # If we've never retrieved the properties before, or the caller is
727 # forcing a check of the server, then check the server
728 if not self._properties or force:
729 path = 'accounts/self' if self._is_pre_162 else 'portals/self'
730 resp = self.con.post(path, self._postdata(), ssl=True)
731 if resp:
732 self._properties = resp
733 self.con.all_ssl = self.is_all_ssl()
734
735 # Return a defensive copy
736 return copy.deepcopy(self._properties)
737
738 def get_user(self, username):
739 """ Returns the user information for the specified username.
740
741 Arguments
742 username required string, the username whose information you want.
743
744 :return:
745 None if the user is not found and returns a dictionary object if the user is found
746 the dictionary has the following keys:
747
748 ================ ========================================================
749 **Key** **Value**
750 ---------------- --------------------------------------------------------
751 access string
752 ---------------- --------------------------------------------------------
753 created time (int)
754 ---------------- --------------------------------------------------------
755 culture string, two-letter language code ('en')
756 ---------------- --------------------------------------------------------
757 description string
758 ---------------- --------------------------------------------------------
759 email string
760 ---------------- --------------------------------------------------------
761 fullName string
762 ---------------- --------------------------------------------------------
763 idpUsername string, name of the user in the enterprise system
764 ---------------- --------------------------------------------------------
765 groups list of dictionaries. For dictionary keys, see get_group doc.
766 ---------------- --------------------------------------------------------
767 modified time (int)
768 ---------------- --------------------------------------------------------
769 orgId string, the organization id
770 ---------------- --------------------------------------------------------
771 preferredView string, value is either Web, GIS, or null
772 ---------------- --------------------------------------------------------
773 region string, None or two letter country code
774 ---------------- --------------------------------------------------------
775 role string, value is either org_user, org_publisher, org_admin
776 ---------------- --------------------------------------------------------
777 storageUsage int
778 ---------------- --------------------------------------------------------
779 storageQuota int
780 ---------------- --------------------------------------------------------
781 tags list of strings
782 ---------------- --------------------------------------------------------
783 thumbnail string, name of file
784 ---------------- --------------------------------------------------------
785 username string, name of user
786 ================ ========================================================
787 """
788 return self.con.post('community/users/' + username, self._postdata())
789
790
791
792
793
794 def invite_group_users(self, user_names, group_id,
795 role='group_member', expiration=10080):
796 """ Invites users to a group.
797
798 .. note::
799 A user who is invited to a group will see a list of invitations
800 in the "Groups" tab of portal listing invitations. The user
801 can either accept or reject the invitation.
802
803 Requires
804 The user executing the command must be group owner
805
806 ================ ========================================================
807 **Argument** **Description**
808 ---------------- --------------------------------------------------------
809 user_names: a required string list of users to invite
810 ---------------- --------------------------------------------------------
811 group_id : required string, specifies the group you are inviting users to.
812 ---------------- --------------------------------------------------------
813 role: an optional string, either group_member or group_admin
814 ---------------- --------------------------------------------------------
815 expiration: an optional int, specifies how long the invitation is valid for in minutes.
816 ================ ========================================================
817
818 :return:
819 a boolean that indicates whether the call succeeded.
820
821 """
822
823 user_names = _unpack(user_names, 'username')
824
825 # Send out the invitations
826 postdata = self._postdata()
827 postdata['users'] = ','.join(user_names)
828 postdata['role'] = role
829 postdata['expiration'] = expiration
830 resp = self.con.post('community/groups/' + group_id + '/invite',
831 postdata)
832
833 if resp:
834 return resp.get('success')
835
836
837 def is_logged_in(self):
838 """ Returns true if logged into the portal. """
839 return self.con.is_logged_in()
840
841 def is_all_ssl(self):
842 """ Returns true if this portal requires SSL. """
843
844 # If properties aren't set yet, return true (assume SSL until the
845 # properties tell us otherwise)
846 if not self._properties:
847 return True
848
849 # If access property doesnt exist, will correctly return false
850 return self._properties.get('allSSL')
851
852 def is_multitenant(self):
853 """ Returns true if this portal is multitenant. """
854 return self._properties['portalMode'] == 'multitenant'
855
856 def is_arcgisonline(self):
857 """ Returns true if this portal is ArcGIS Online. """
858 return self._properties['portalName'] == 'ArcGIS Online' \
859 and self.is_multitenant()
860
861 def is_subscription(self):
862 """ Returns true if this portal is an ArcGIS Online subscription. """
863 return bool(self._properties.get('urlKey'))
864
865 def is_org(self):
866 """ Returns true if this portal is an organization. """
867 return bool(self._properties.get('id'))
868
869
870 def leave_group(self, group_id):
871 """ Removes the logged in user from the specified group.
872
873 Requires:
874 User must be logged in.
875
876 Arguments:
877 group_id: required string, specifies the group id
878
879 :return:
880 a boolean indicating whether the operation was successful.
881 """
882 resp = self.con.post('community/groups/' + group_id + '/leave',
883 self._postdata())
884 if resp:
885 return resp.get('success')
886
887 def login(self, username, password, expiration=60):
888 """ Logs into the portal using username/password.
889
890 .. note::
891 You can log into a portal when you construct a portal
892 object or you can login later. This function is
893 for the situation when you need to log in later.
894
895 ================ ========================================================
896 **Argument** **Description**
897 ---------------- --------------------------------------------------------
898 username required string
899 ---------------- --------------------------------------------------------
900 password required string
901 ---------------- --------------------------------------------------------
902 expiration optional int, how long the token generated should last.
903 ================ ========================================================
904
905 :return:
906 a string, the token
907
908 """
909
910 newtoken = self.con.login(username, password, expiration)
911 if newtoken:
912 self._logged_in_user = self.get_user(username)
913 return newtoken
914
915 def logout(self):
916 """ Logs out of the portal.
917
918 .. note::
919 The portal will forget any existing tokens it was using, all
920 subsequent portal calls will be anonymous until another login
921 call occurs.
922
923 :return:
924 No return value.
925
926 """
927
928 self.con.logout()
929
930
931 def logged_in_user(self):
932 """ Returns information about the logged in user.
933
934 :return:
935 a dict with the following keys:
936
937 ================ ========================================================
938 **Key** **Value**
939 ---------------- --------------------------------------------------------
940 username string
941 ---------------- --------------------------------------------------------
942 storageUsage int
943 ---------------- --------------------------------------------------------
944 description string
945 ---------------- --------------------------------------------------------
946 tags comma-separated string
947 ---------------- --------------------------------------------------------
948 created int, when group created (ms since 1 Jan 1970)
949 ---------------- --------------------------------------------------------
950 modified int, when group last modified (ms since 1 Jan 1970)
951 ---------------- --------------------------------------------------------
952 fullName string
953 ---------------- --------------------------------------------------------
954 email string
955 ---------------- --------------------------------------------------------
956 idpUsername string, name of the user in their identity provider
957 ================ ========================================================
958
959 """
960 if self._logged_in_user:
961 # Return a defensive copy
962 return copy.deepcopy(self._logged_in_user)
963 return None
964
965
966 def reassign_user(self, username, target_username):
967 """ Reassigns all of a user's items and groups to another user.
968
969 Items are transferred to the target user into a folder named
970 <user>_<folder> where user corresponds to the user whose items were
971 moved and folder corresponds to the folder that was moved.
972
973 .. note::
974 This method must be executed as an administrator. This method also
975 can not be undone. The changes are immediately made and permanent.
976
977 ================ ========================================================
978 **Argument** **Description**
979 ---------------- --------------------------------------------------------
980 username required string, user who will have items/groups transferred
981 ---------------- --------------------------------------------------------
982 target_username required string, user who will own items/groups after this.
983 ================ ========================================================
984
985 :return:
986 a boolean indicating success
987
988 """
989
990 postdata = self._postdata()
991 postdata['targetUsername'] = target_username
992 resp = self.con.post('community/users/' + username + '/reassign', postdata)
993 if resp:
994 return resp.get('success')
995
996
997
998 def reassign_group(self, group_id, target_owner):
999 """ Reassigns a group to another owner.
1000
1001
1002
1003 ================ ========================================================
1004 **Argument** **Description**
1005 ---------------- --------------------------------------------------------
1006 group_id required string, unique identifier for the group
1007 ---------------- --------------------------------------------------------
1008 target_owner required string, username of new group owner
1009 ================ ========================================================
1010
1011 :return:
1012 a boolean, indicating success
1013
1014 """
1015 postdata = self._postdata()
1016 postdata['targetUsername'] = target_owner
1017 resp = self.con.post('community/groups/' + group_id + '/reassign', postdata)
1018 if resp:
1019 return resp.get('success')
1020
1021
1022 def reassign_item(self, item_id, current_owner, target_owner, current_folder=None, target_folder=None):
1023 """ Allows the administrator to reassign a single item from one user to another.
1024
1025 .. note::
1026 If you wish to move all of a user's items (and groups) to another user then use the
1027 reassign_user method. This method only moves one item at a time.
1028
1029 ================ ========================================================
1030 **Argument** **Description**
1031 ---------------- --------------------------------------------------------
1032 item_id required string, unique identifier for the item
1033 ---------------- --------------------------------------------------------
1034 current_owner required string, owner of the item currently
1035 ---------------- --------------------------------------------------------
1036 current_folder optional string, folder containing the item. Defaults to the root folder.
1037 ---------------- --------------------------------------------------------
1038 target_owner required string, desired owner of the item
1039 ---------------- --------------------------------------------------------
1040 target_folder optional string, folder to move the item to.
1041 ================ ========================================================
1042
1043 :return:
1044 a boolean, indicating success
1045
1046 """
1047
1048 path = '/content/users/' + current_owner
1049 if current_folder :
1050 path += '/folder'
1051 path += 'items/' + item_id + '/reassign'
1052
1053 postdata = self._postdata()
1054 postdata['targetUsername'] = target_owner
1055 postdata['targetFoldername'] = target_folder if target_folder else '/'
1056 return self.con.post(path, postdata)
1057
1058
1059
1060 def reset_user(self, username, password, new_password=None,
1061 new_security_question=None, new_security_answer=None):
1062 """ Resets a user's password, security question, and/or security answer.
1063
1064 .. note::
1065 This function does not apply to those using enterprise accounts
1066 that come from an enterprise such as ActiveDirectory, LDAP, or SAML.
1067 It only has an effect on built-in users.
1068
1069 If a new security question is specified, a new security answer should
1070 be provided.
1071
1072 ===================== ========================================================
1073 **Argument** **Description**
1074 --------------------- --------------------------------------------------------
1075 username required string, account being reset
1076 --------------------- --------------------------------------------------------
1077 password required string, current password
1078 --------------------- --------------------------------------------------------
1079 new_password optional string, new password if resetting password
1080 --------------------- --------------------------------------------------------
1081 new_security_question optional int, new security question if desired
1082 --------------------- --------------------------------------------------------
1083 new_security_answer optional string, new security question answer if desired
1084 ===================== ========================================================
1085
1086 :return:
1087 a boolean, indicating success
1088
1089 """
1090 postdata = self._postdata()
1091 postdata['password'] = password
1092 if new_password:
1093 postdata['newPassword'] = new_password
1094 if new_security_question:
1095 postdata['newSecurityQuestionIdx'] = new_security_question
1096 if new_security_answer:
1097 postdata['newSecurityAnswer'] = new_security_answer
1098 resp = self.con.post('community/users/' + username + '/reset',
1099 postdata, ssl=True)
1100 if resp:
1101 return resp.get('success')
1102
1103
1104
1105 def remove_group_users(self, user_names, group_id):
1106 """ Remove users from a group.
1107
1108 ================ ========================================================
1109 **Argument** **Description**
1110 ---------------- --------------------------------------------------------
1111 user_names required string, comma-separated list of users
1112 ---------------- --------------------------------------------------------
1113 group_id required string, the id for a group.
1114 ================ ========================================================
1115
1116 :return:
1117 a dictionary with a key notRemoved that is a list of users not removed.
1118
1119 """
1120
1121 user_names = _unpack(user_names, 'username')
1122
1123 # Remove the users from the group
1124 postdata = self._postdata()
1125 postdata['users'] = ','.join(user_names)
1126 resp = self.con.post('community/groups/' + group_id + '/removeUsers',
1127 postdata)
1128 return resp
1129
1130
1131 def search(self, q, bbox=None, sort_field='title', sort_order='asc',
1132 max_results=1000, add_org=True):
1133
1134
1135 if add_org:
1136 accountid = self._properties.get('id')
1137 if accountid and q:
1138 q += ' accountid:' + accountid
1139 elif accountid:
1140 q = 'accountid:' + accountid
1141
1142 count = 0
1143 resp = self._search_page(q, bbox, 1, min(max_results, 100), sort_field, sort_order)
1144 results = resp.get('results')
1145 count += int(resp['num'])
1146 nextstart = int(resp['nextStart'])
1147 while count < max_results and nextstart > 0:
1148 resp = self._search_page(q, bbox, nextstart, min(max_results - count, 100),
1149 sort_field, sort_order)
1150 results.extend(resp['results'])
1151 count += int(resp['num'])
1152 nextstart = int(resp['nextStart'])
1153
1154 return results
1155
1156
1157 def search_groups(self, q, sort_field='title',sort_order='asc',
1158 max_groups=1000, add_org=True):
1159 """ Searches for portal groups.
1160
1161 .. note::
1162 A few things that will be helpful to know.
1163
1164 1. The query syntax has quite a few features that can't
1165 be adequately described here. The query syntax is
1166 available in ArcGIS help. A short version of that URL
1167 is http://bitly.com/1fJ8q31.
1168
1169 2. Most of the time when searching groups you want to
1170 search within your organization in ArcGIS Online
1171 or within your Portal. As a convenience, the method
1172 automatically appends your organization id to the query by
1173 default. If you don't want the API to append to your query
1174 set add_org to false.
1175
1176 ================ ========================================================
1177 **Argument** **Description**
1178 ---------------- --------------------------------------------------------
1179 q required string, query string. See notes.
1180 ---------------- --------------------------------------------------------
1181 sort_field optional string, valid values can be title, owner, created
1182 ---------------- --------------------------------------------------------
1183 sort_order optional string, valid values are asc or desc
1184 ---------------- --------------------------------------------------------
1185 max_groups optional int, maximum number of groups returned
1186 ---------------- --------------------------------------------------------
1187 add_org optional boolean, controls whether to search within your org
1188 ================ ========================================================
1189
1190 :return:
1191 A list of dictionaries. Each dictionary has the following keys.
1192
1193 ================ ========================================================
1194 **Key** **Value**
1195 ---------------- --------------------------------------------------------
1196 access string, values=private, org, public
1197 ---------------- --------------------------------------------------------
1198 created int, ms since 1 Jan 1970
1199 ---------------- --------------------------------------------------------
1200 description string
1201 ---------------- --------------------------------------------------------
1202 id string, unique id for group
1203 ---------------- --------------------------------------------------------
1204 isInvitationOnly boolean
1205 ---------------- --------------------------------------------------------
1206 isViewOnly boolean
1207 ---------------- --------------------------------------------------------
1208 modified int, ms since 1 Jan 1970
1209 ---------------- --------------------------------------------------------
1210 owner string, user name of owner
1211 ---------------- --------------------------------------------------------
1212 phone string
1213 ---------------- --------------------------------------------------------
1214 snippet string, short summary of group
1215 ---------------- --------------------------------------------------------
1216 sortField string, how shared items are sorted
1217 ---------------- --------------------------------------------------------
1218 sortOrder string, asc or desc
1219 ---------------- --------------------------------------------------------
1220 tags string list, user supplied tags for searching
1221 ---------------- --------------------------------------------------------
1222 thumbnail string, name of file. Append to http://<community url>/groups/<group id>/info/
1223 ---------------- --------------------------------------------------------
1224 title string, name of group as shown to users
1225 ================ ========================================================
1226 """
1227
1228 if add_org:
1229 accountid = self._properties.get('id')
1230 if accountid and q:
1231 q += ' accountid:' + accountid
1232 elif accountid:
1233 q = 'accountid:' + accountid
1234
1235 # Execute the search and get back the results
1236 count = 0
1237 resp = self._groups_page(q, 1, min(max_groups,100), sort_field, sort_order)
1238 results = resp.get('results')
1239 count += int(resp['num'])
1240 nextstart = int(resp['nextStart'])
1241 while count < max_groups and nextstart > 0:
1242 resp = self._groups_page(q, 1, min(max_groups - count,100),
1243 sort_field, sort_order)
1244 resp_users = resp.get('results')
1245 results.extend(resp_users)
1246 count += int(resp['num'])
1247 nextstart = int(resp['nextStart'])
1248
1249 return results
1250
1251
1252
1253 def search_users(self, q, sort_field='username',
1254 sort_order='asc', max_users=1000, add_org=True):
1255 """ Searches portal users.
1256
1257 This gives you a list of users and some basic information
1258 about those users. To get more detailed information (such as role), you
1259 may need to call get_user on each user.
1260
1261 .. note::
1262 A few things that will be helpful to know.
1263
1264 1. The query syntax has quite a few features that can't
1265 be adequately described here. The query syntax is
1266 available in ArcGIS help. A short version of that URL
1267 is http://bitly.com/1fJ8q31.
1268
1269 2. Most of the time when searching groups you want to
1270 search within your organization in ArcGIS Online
1271 or within your Portal. As a convenience, the method
1272 automatically appends your organization id to the query by
1273 default. If you don't want the API to append to your query
1274 set add_org to false. If you use this feature with an
1275 OR clause such as field=x or field=y you should put this
1276 into parenthesis when using add_org.
1277
1278 ================ ========================================================
1279 **Argument** **Description**
1280 ---------------- --------------------------------------------------------
1281 q required string, query string. See notes.
1282 ---------------- --------------------------------------------------------
1283 sort_field optional string, valid values can be username or created
1284 ---------------- --------------------------------------------------------
1285 sort_order optional string, valid values are asc or desc
1286 ---------------- --------------------------------------------------------
1287 max_users optional int, maximum number of users returned
1288 ---------------- --------------------------------------------------------
1289 add_org optional boolean, controls whether to search within your org
1290 ================ ========================================================
1291
1292 :return:
1293 A a list of dictionary objects with the following keys:
1294
1295 ================ ========================================================
1296 **Key** **Value**
1297 ---------------- --------------------------------------------------------
1298 created time (int), when user created
1299 ---------------- --------------------------------------------------------
1300 culture string, two-letter language code
1301 ---------------- --------------------------------------------------------
1302 description string, user supplied description
1303 ---------------- --------------------------------------------------------
1304 fullName string, name of the user
1305 ---------------- --------------------------------------------------------
1306 modified time (int), when user last modified
1307 ---------------- --------------------------------------------------------
1308 region string, may be None
1309 ---------------- --------------------------------------------------------
1310 tags string list, of user tags
1311 ---------------- --------------------------------------------------------
1312 thumbnail string, name of file
1313 ---------------- --------------------------------------------------------
1314 username string, name of the user
1315 ================ ========================================================
1316 """
1317
1318 if add_org:
1319 accountid = self._properties.get('id')
1320 if accountid and q:
1321 q += ' accountid:' + accountid
1322 elif accountid:
1323 q = 'accountid:' + accountid
1324
1325 # Execute the search and get back the results
1326 count = 0
1327 resp = self._users_page(q, 1, min(max_users, 100), sort_field, sort_order)
1328 results = resp.get('results')
1329 count += int(resp['num'])
1330 nextstart = int(resp['nextStart'])
1331 while count < max_users and nextstart > 0:
1332 resp = self._users_page(q, nextstart, min(max_users - count, 100),
1333 sort_field, sort_order)
1334 resp_users = resp.get('results')
1335 results.extend(resp_users)
1336 count += int(resp['num'])
1337 nextstart = int(resp['nextStart'])
1338
1339 return results
1340
1341
1342
1343 # Used to signup a new user to an on-premises portal.
1344 def signup(self, username, password, fullname, email):
1345 """ Signs up users to an instance of Portal for ArcGIS.
1346
1347 .. note::
1348 This method only applies to Portal and not ArcGIS
1349 Online. This method can be called anonymously, but
1350 keep in mind that self-signup can also be disabled
1351 in a Portal. It also only creates built-in
1352 accounts, it does not work with enterprise
1353 accounts coming from ActiveDirectory or your
1354 LDAP.
1355
1356 There is another method called createUser that
1357 requires administrator access that can always
1358 be used against 10.2.1 portals or later that
1359 can create users whether they are builtin or
1360 enterprise accounts.
1361
1362 ================ ========================================================
1363 **Argument** **Description**
1364 ---------------- --------------------------------------------------------
1365 username required string, must be unique in the Portal, >4 characters
1366 ---------------- --------------------------------------------------------
1367 password required string, must be >= 8 characters.
1368 ---------------- --------------------------------------------------------
1369 fullname required string, name of the user
1370 ---------------- --------------------------------------------------------
1371 email required string, must be an email address
1372 ================ ========================================================
1373
1374 :return:
1375 a boolean indicating success
1376
1377 """
1378 if self.is_arcgisonline():
1379 raise ValueError('Signup is not supported on ArcGIS Online')
1380
1381 postdata = self._postdata()
1382 postdata['username'] = username
1383 postdata['password'] = password
1384 postdata['fullname'] = fullname
1385 postdata['email'] = email
1386 resp = self.con.post('community/signUp', postdata, ssl=True)
1387 if resp:
1388 return resp.get('success')
1389
1390
1391 def update_user(self, username, access=None, preferred_view=None,
1392 description=None, tags=None, thumbnail=None,
1393 fullname=None, email=None, culture=None,
1394 region=None):
1395 """ Updates a user's properties.
1396
1397 .. note::
1398 Only pass in arguments for properties you want to update.
1399 All other properties will be left as they are. If you
1400 want to update description, then only provide
1401 the description argument.
1402
1403 ================ ========================================================
1404 **Argument** **Description**
1405 ---------------- --------------------------------------------------------
1406 username required string, name of the user to be updated.
1407 ---------------- --------------------------------------------------------
1408 access optional string, values: private, org, public
1409 ---------------- --------------------------------------------------------
1410 preferred_view optional string, values: Web, GIS, null
1411 ---------------- --------------------------------------------------------
1412 description optional string, a description of the user.
1413 ---------------- --------------------------------------------------------
1414 tags optional string, comma-separated tags for searching
1415 ---------------- --------------------------------------------------------
1416 thumbnail optional string, path or url to a file. can be PNG, GIF,
1417 JPEG, max size 1 MB
1418 ---------------- --------------------------------------------------------
1419 fullname optional string, name of the user, only for built-in users
1420 ---------------- --------------------------------------------------------
1421 email optional string, email address, only for built-in users
1422 ---------------- --------------------------------------------------------
1423 culture optional string, two-letter language code, fr for example
1424 ---------------- --------------------------------------------------------
1425 region optional string, two-letter country code, FR for example
1426 ================ ========================================================
1427
1428 :return:
1429 a boolean indicating success
1430
1431 """
1432 properties = dict()
1433 postdata = self._postdata()
1434 if access:
1435 properties['access'] = access
1436 if preferred_view:
1437 properties['preferredView'] = preferred_view
1438 if description:
1439 properties['description'] = description
1440 if tags:
1441 properties['tags'] = tags
1442 if fullname:
1443 properties['fullname'] = fullname
1444 if email:
1445 properties['email'] = email
1446 if culture:
1447 properties['culture'] = culture
1448 if region:
1449 properties['region'] = region
1450
1451 files = []
1452 if thumbnail:
1453 if _is_http_url(thumbnail):
1454 thumbnail = urllib.urlretrieve(thumbnail)[0]
1455 file_ext = os.path.splitext(thumbnail)[1]
1456 if not file_ext:
1457 file_ext = imghdr.what(thumbnail)
1458 if file_ext in ('gif', 'png', 'jpeg'):
1459 new_thumbnail = thumbnail + '.' + file_ext
1460 os.rename(thumbnail, new_thumbnail)
1461 thumbnail = new_thumbnail
1462 files.append(('thumbnail', thumbnail, os.path.basename(thumbnail)))
1463 postdata.update(properties)
1464
1465
1466 # Send the POST request, and return the id from the response
1467 resp = self.con.post('community/users/' + username + '/update', postdata, files, ssl=True)
1468
1469
1470 if resp:
1471 return resp.get('success')
1472
1473
1474
1475 def update_user_role(self, username, role):
1476 """ Updates a user's role.
1477
1478 .. note::
1479 There are three types of roles in Portal - user, publisher, and administrator.
1480 A user can share items, create maps, create groups, etc. A publisher can
1481 do everything a user can do and create hosted services. An administrator can
1482 do everything that is possible in Portal.
1483
1484 ================ ========================================================
1485 **Argument** **Description**
1486 ---------------- --------------------------------------------------------
1487 username required string, the name of the user whose role will change
1488 ---------------- --------------------------------------------------------
1489 role required string, one of these values org_user, org_publisher, org_admin
1490 ================ ========================================================
1491
1492 :return:
1493 a boolean, that indicates success
1494
1495 """
1496 postdata = self._postdata()
1497 postdata.update({'user': username, 'role': role})
1498 resp = self.con.post('portals/self/updateuserrole', postdata, ssl=True)
1499 if resp:
1500 return resp.get('success')
1501
1502
1503 def update_group(self, group_id, title=None, tags=None, description=None,
1504 snippet=None, access=None, is_invitation_only=None,
1505 sort_field=None, sort_order=None, is_view_only=None,
1506 thumbnail=None):
1507 """ Updates a group.
1508
1509 .. note::
1510 Only provide the values for the arguments you wish to update.
1511
1512 ================== ========================================================
1513 **Argument** **Description**
1514 ------------------ --------------------------------------------------------
1515 group_id required string, the group to modify
1516 ------------------ --------------------------------------------------------
1517 title optional string, name of the group
1518 ------------------ --------------------------------------------------------
1519 tags optional string, comma-delimited list of tags
1520 ------------------ --------------------------------------------------------
1521 description optional string, describes group in detail
1522 ------------------ --------------------------------------------------------
1523 snippet optional string, <250 characters summarizes group
1524 ------------------ --------------------------------------------------------
1525 access optional string, can be private, public, or org
1526 ------------------ --------------------------------------------------------
1527 thumbnail optional string, URL or file location to group image
1528 ------------------ --------------------------------------------------------
1529 is_invitation_only optional boolean, defines whether users can join by request.
1530 ------------------ --------------------------------------------------------
1531 sort_field optional string, specifies how shared items with the group are sorted.
1532 ------------------ --------------------------------------------------------
1533 sort_order optional string, asc or desc for ascending or descending.
1534 ------------------ --------------------------------------------------------
1535 is_view_only optional boolean, defines whether the group is searchable
1536 ================== ========================================================
1537
1538 :return:
1539 a boolean indicating success
1540 """
1541
1542
1543
1544 properties = dict()
1545 postdata = self._postdata()
1546 if title:
1547 properties['title'] = title
1548 if tags:
1549 properties['tags'] = tags
1550 if description:
1551 properties['description'] = description
1552 if snippet:
1553 properties['snippet'] = snippet
1554 if access:
1555 properties['access'] = access
1556 if is_invitation_only:
1557 properties['isinvitationOnly'] = is_invitation_only
1558 if sort_field:
1559 properties['sortField'] = sort_field
1560 if sort_order:
1561 properties['sortOrder'] = sort_order
1562 if is_view_only:
1563 properties['isViewOnly'] = is_view_only
1564
1565 postdata.update(properties)
1566
1567 files = []
1568 if thumbnail:
1569 if _is_http_url(thumbnail):
1570 thumbnail = urllib.urlretrieve(thumbnail)[0]
1571 file_ext = os.path.splitext(thumbnail)[1]
1572 if not file_ext:
1573 file_ext = imghdr.what(thumbnail)
1574 if file_ext in ('gif', 'png', 'jpeg'):
1575 new_thumbnail = thumbnail + '.' + file_ext
1576 os.rename(thumbnail, new_thumbnail)
1577 thumbnail = new_thumbnail
1578 files.append(('thumbnail', thumbnail, os.path.basename(thumbnail)))
1579
1580 resp = self.con.post('community/groups/' + group_id + '/update', postdata, files)
1581 if resp:
1582 return resp.get('success')
1583
1584
1585
1586 def get_version(self, force=False):
1587 """ Returns the portal version (using cache unless force=True).
1588
1589 .. note::
1590 The version information is retrieved when you create the
1591 Portal object and then cached for future requests. If you
1592 want to make a request to the Portal and not rely on the
1593 cache then you can set the force argument to True.
1594
1595 Arguments:
1596 force boolean, true=make a request, false=use cache
1597
1598 :return:
1599 a string with the version. The version is an internal number
1600 that may not match the version of the product purchased. So
1601 2.3 is returned from Portal 10.2.1 for instance.
1602
1603
1604 """
1605
1606 # If we've never retrieved the version before, or the caller is
1607 # forcing a check of the server, then check the server
1608 if not self._version or force:
1609 resp = self.con.post('', self._postdata())
1610 if not resp:
1611 old_resturl = _normalize_url(self.url) + 'sharing/'
1612 resp = self.con.post(old_resturl, self._postdata(), ssl=True)
1613 if resp:
1614 _log.warn('Portal is pre-1.6.2; some things may not work')
1615 self._is_pre_162 = True
1616 self._is_pre_21 = True
1617 self.resturl = old_resturl
1618 self.con.baseurl = old_resturl
1619 else:
1620 version = resp.get('currentVersion')
1621 if version == '1.6.2' or version == '2.0':
1622 _log.warn('Portal is pre-2.1; some features not supported')
1623 self._is_pre_21 = True
1624 if resp:
1625 self._version = resp.get('currentVersion')
1626
1627 return self._version
1628
1629
1630 def create_folder(self, owner, title):
1631 """ Creates a folder for the given user with the given title.
1632
1633 ================ ========================================================
1634 **Argument** **Description**
1635 ---------------- --------------------------------------------------------
1636 owner required string, the name of the user
1637 ---------------- --------------------------------------------------------
1638 title required string, the name of the folder to create for the owner
1639 ================ ========================================================
1640
1641 :return:
1642 a json object like the following:
1643 {"username" : "portaladmin","id" : "bff13218991c4485a62c81db3512396f","title" : "testcreate"}
1644 """
1645 postdata = self._postdata()
1646 postdata['title'] = title
1647 resp = self.con.post('content/users/' + owner + '/createFolder', postdata)
1648 if resp and resp.get('success'):
1649 return resp['folder']
1650
1651
1652
1653 def delete_folder(self, owner, folder_id):
1654 """ Creates a folder for the given user with the given title.
1655
1656 ================ ========================================================
1657 **Argument** **Description**
1658 ---------------- --------------------------------------------------------
1659 owner required string, the name of the user
1660 ---------------- --------------------------------------------------------
1661 folder_id required string, the id of the folder
1662 ================ ========================================================
1663
1664 :return:
1665 a boolean if succeeded.
1666 """
1667 postdata = self._postdata()
1668 resp = self.con.post('content/users/' + owner + '/' + folder_id + '/delete', postdata)
1669 if resp:
1670 return resp.get('success')
1671
1672
1673
1674 def get_folder_id(self, owner, folder_name):
1675 """ Finds the folder for a particular owner and returns its id.
1676
1677 ================ ========================================================
1678 **Argument** **Description**
1679 ---------------- --------------------------------------------------------
1680 owner required string, the name of the user
1681 ---------------- --------------------------------------------------------
1682 folder_name required string, the name of the folder to search for
1683 ================ ========================================================
1684
1685 :return:
1686 a boolean if succeeded.
1687 """
1688 resp = self.con.post('content/users/' + owner, self._postdata())
1689 if resp and 'folders' in resp:
1690 # Loop through each folder JSON object
1691 for fldr in resp['folders']:
1692 if fldr['title'].upper() == folder_name.upper(): # Force both strings to upper case for comparison
1693 return fldr['id']
1694
1695
1696
1697 def _is_searching_public(self, scope):
1698 if scope == 'public':
1699 return True
1700 elif scope == 'org':
1701 return False
1702 elif scope == 'default' or scope is None:
1703 # By default orgs won't search public
1704 return False if self.is_org() else True
1705 else:
1706 raise ValueError('Unknown scope "' + scope + '". Supported ' \
1707 + 'values are "public", "org", and "default"')
1708
1709
1710 def _invitations_page(self, start, num):
1711 postdata = self._postdata()
1712 postdata.update({ 'start': start, 'num': num })
1713 return self.con.post('portals/self/invitations', postdata)
1714
1715
1716
1717 def _postdata(self):
1718 if self._basepostdata:
1719 # Return a defensive copy
1720 return copy.deepcopy(self._basepostdata)
1721 return None
1722
1723
1724
1725 def _search_page(self, q=None, bbox=None, start=1, num=10, sortfield='', sortorder='asc'):
1726 _log.info('Searching items (q=' + str(q) + ', bbox=' + str(bbox) \
1727 + ', start=' + str(start) + ', num=' + str(num) + ')')
1728 postdata = self._postdata()
1729 postdata.update({ 'q': q or '', 'bbox': bbox or '', 'start': start, 'num': num,
1730 'sortField': sortfield, 'sortOrder': sortorder })
1731 return self.con.post('search', postdata)
1732
1733
1734 def _groups_page(self, q=None, start=1, num=10, sortfield='',
1735 sortorder='asc'):
1736 _log.info('Searching groups (q=' + str(q) + ', start=' + str(start) \
1737 + ', num=' + str(num) + ')')
1738 postdata = self._postdata()
1739 postdata.update({ 'q': q, 'start': start, 'num': num,
1740 'sortField': sortfield, 'sortOrder': sortorder })
1741 return self.con.post('community/groups', postdata)
1742
1743
1744 def _org_users_page(self, start=1, num=10):
1745 _log.info('Retrieving org users (start=' + str(start) \
1746 + ', num=' + str(num) + ')')
1747 postdata = self._postdata()
1748 postdata['start'] = start
1749 postdata['num'] = num
1750 return self.con.post('portals/self/users', postdata)
1751
1752
1753 def _users_page(self, q=None, start=1, num=10, sortfield='', sortorder='asc'):
1754 _log.info('Searching users (q=' + str(q) + ', start=' + str(start) \
1755 + ', num=' + str(num) + ')')
1756 postdata = self._postdata()
1757 postdata.update({ 'q': q, 'start': start, 'num': num,
1758 'sortField': sortfield, 'sortOrder': sortorder })
1759 return self.con.post('community/users', postdata)
1760
1761
1762
1763
1764 def _extract(self, results, props=None):
1765 if not props or len(props) == 0:
1766 return results
1767 newresults = []
1768 for result in results:
1769 newresult = dict((p, result[p]) for p in props if p in result)
1770 newresults.append(newresult)
1771 return newresults
1772
1773class _ArcGISConnection(object):
1774 """ A class users to manage connection to ArcGIS services (Portal and Server). """
1775
1776 def __init__(self, baseurl, username=None, password=None, key_file=None,
1777 cert_file=None, expiration=60, all_ssl=False, referer=None,
1778 proxy_host=None, proxy_port=None, ensure_ascii=True):
1779 """ The _ArcGISConnection constructor. Requires URL and optionally username/password. """
1780
1781 self.baseurl = _normalize_url(baseurl)
1782 self.key_file = key_file
1783 self.cert_file = cert_file
1784 self.all_ssl = all_ssl
1785 self.proxy_host = proxy_host
1786 self.proxy_port = proxy_port
1787 self.ensure_ascii = ensure_ascii
1788 self.token = None
1789
1790 # Setup the referer and user agent
1791 if not referer:
1792 import socket
1793 ip = socket.gethostbyname(socket.gethostname())
1794 referer = socket.gethostbyaddr(ip)[0]
1795 self._referer = referer
1796 self._useragent = 'PortalPy/' + __version__
1797
1798 # Login if credentials were provided
1799 if username and password:
1800 self.login(username, password, expiration)
1801 elif username or password:
1802 _log.warning('Both username and password required for login')
1803
1804 def generate_token(self, username, password, expiration=60):
1805 """ Generates and returns a new token, but doesn't re-login. """
1806 postdata = { 'username': username, 'password': password,
1807 'client': 'referer', 'referer': self._referer,
1808 'expiration': expiration, 'f': 'json' }
1809 resp = self.post('generateToken', postdata, ssl=True)
1810 if resp:
1811 return resp.get('token')
1812
1813 def login(self, username, password, expiration=60):
1814 """ Logs into the portal using username/password. """
1815 newtoken = self.generate_token(username, password, expiration)
1816 if newtoken:
1817 self.token = newtoken
1818 self._username = username
1819 self._password = password
1820 self._expiration = expiration
1821 return newtoken
1822
1823 def relogin(self, expiration=None):
1824 """ Re-authenticates with the portal using the same username/password. """
1825 if not expiration:
1826 expiration = self._expiration
1827 return self.login(self._username, self._password, expiration)
1828
1829 def logout(self):
1830 """ Logs out of the portal. """
1831 self.token = None
1832
1833 def is_logged_in(self):
1834 """ Returns true if logged into the portal. """
1835 return self.token is not None
1836
1837 def get(self, path, ssl=False, compress=True, try_json=True, is_retry=False):
1838 """ Returns result of an HTTP GET. Handles token timeout and all SSL mode."""
1839 url = path
1840 if not path.startswith('http://') and not path.startswith('https://'):
1841 url = self.baseurl + path
1842 if ssl or self.all_ssl:
1843 url = url.replace('http://', 'https://')
1844
1845 # Add the token if logged in
1846 if self.is_logged_in():
1847 url = self._url_add_token(url, self.token)
1848
1849 _log.debug('REQUEST (get): ' + url)
1850
1851 try:
1852 # Send the request and read the response
1853 headers = [('Referer', self._referer),
1854 ('User-Agent', self._useragent)]
1855 if compress:
1856 headers.append(('Accept-encoding', 'gzip'))
1857 opener = urllib2.build_opener()
1858 opener.addheaders = headers
1859 resp = opener.open(url)
1860 if resp.info().get('Content-Encoding') == 'gzip':
1861 buf = StringIO(resp.read())
1862 f = gzip.GzipFile(fileobj=buf)
1863 resp_data = f.read()
1864 else:
1865 resp_data = resp.read()
1866
1867 # If we're not trying to parse to JSON, return response as is
1868 if not try_json:
1869 return resp_data
1870
1871 try:
1872 resp_json = json.loads(resp_data)
1873
1874 # Convert to ascii if directed to do so
1875 if self.ensure_ascii:
1876 resp_json = _unicode_to_ascii(resp_json)
1877
1878 # Check for errors, and handle the case where the token timed
1879 # out during use (and simply needs to be re-generated)
1880 try:
1881 if resp_json.get('error', None):
1882 errorcode = resp_json['error']['code']
1883 if errorcode == 498 and not is_retry:
1884 _log.info('Token expired during get request, ' \
1885 + 'fetching a new token and retrying')
1886 newtoken = self.relogin()
1887 newpath = self._url_add_token(path, newtoken)
1888 return self.get(newpath, ssl, compress, try_json, is_retry=True)
1889 elif errorcode == 498:
1890 raise RuntimeError('Invalid token')
1891 self._handle_json_error(resp_json['error'])
1892 return None
1893 except AttributeError:
1894 # Top-level JSON object isnt a dict, so can't have an error
1895 pass
1896
1897 # If the JSON parsed correctly and there are no errors,
1898 # return the JSON
1899 return resp_json
1900
1901 # If we couldnt parse the response to JSON, return it as is
1902 except ValueError:
1903 return resp
1904
1905 # If we got an HTTPError when making the request check to see if it's
1906 # related to token timeout, in which case, regenerate a token
1907 except urllib2.HTTPError as e:
1908 if e.code == 498 and not is_retry:
1909 _log.info('Token expired during get request, fetching a new ' \
1910 + 'token and retrying')
1911 self.logout()
1912 newtoken = self.relogin()
1913 newpath = self._url_add_token(path, newtoken)
1914 return self.get(newpath, ssl, try_json, is_retry=True)
1915 elif e.code == 498:
1916 raise RuntimeError('Invalid token')
1917 else:
1918 raise e
1919
1920 def download(self, path, filepath, ssl=False, is_retry=False):
1921 """ Downloads result of an HTTP GET. Handles token timeout and all SSL mode."""
1922 url = path
1923 if not path.startswith('http://') and not path.startswith('https://'):
1924 url = self.baseurl + path
1925 if ssl or self.all_ssl:
1926 url = url.replace('http://', 'https://')
1927
1928 # Add the token if logged in
1929 if self.is_logged_in():
1930 url = self._url_add_token(url, self.token)
1931
1932 _log.debug('REQUEST (download): ' + url + ', to ' + filepath)
1933
1934 # Send the request, and handle the case where the token has
1935 # timed out (relogin and try again)
1936 try:
1937 opener = _StrictURLopener()
1938 opener.addheaders = [('Referer', self._referer),
1939 ('User-Agent', self._useragent)]
1940 opener.retrieve(url, filepath)
1941 except urllib2.HTTPError as e:
1942 if e.code == 498 and not is_retry:
1943 _log.info('Token expired during download request, fetching a ' \
1944 + 'new token and retrying')
1945 self.logout()
1946 newtoken = self.relogin()
1947 newpath = self._url_add_token(path, newtoken)
1948 self.download(newpath, filepath, ssl, is_retry=True)
1949 elif e.code == 498:
1950 raise RuntimeError('Invalid token')
1951 else:
1952 raise e
1953
1954 def _url_add_token(self, url, token):
1955
1956 # Parse the URL and query string
1957 urlparts = urlparse.urlparse(url)
1958 qs_list = urlparse.parse_qsl(urlparts.query)
1959
1960 # Update the token query string parameter
1961 replaced_token = False
1962 new_qs_list = []
1963 for qs_param in qs_list:
1964 if qs_param[0] == 'token':
1965 qs_param = ('token', token)
1966 replaced_token = True
1967 new_qs_list.append(qs_param)
1968 if not replaced_token:
1969 new_qs_list.append(('token', token))
1970
1971 # Rebuild the URL from parts and return it
1972 return urlparse.urlunparse((urlparts.scheme, urlparts.netloc,
1973 urlparts.path, urlparts.params,
1974 urllib.urlencode(new_qs_list),
1975 urlparts.fragment))
1976
1977 def post(self, path, postdata=None, files=None, ssl=False, compress=True,
1978 is_retry=False):
1979 """ Returns result of an HTTP POST. Supports Multipart requests."""
1980 url = path
1981 if not path.startswith('http://') and not path.startswith('https://'):
1982 url = self.baseurl + path
1983 if ssl or self.all_ssl:
1984 url = url.replace('http://', 'https://')
1985
1986 # Add the token if logged in
1987 if self.is_logged_in():
1988 postdata['token'] = self.token
1989
1990 if _log.isEnabledFor(logging.DEBUG):
1991 msg = 'REQUEST: ' + url + ', ' + str(postdata)
1992 if files:
1993 msg += ', files=' + str(files)
1994 _log.debug(msg)
1995
1996 # If there are files present, send a multipart request
1997 if files:
1998 parsed_url = urlparse.urlparse(url)
1999 resp_data = self._postmultipart(parsed_url.netloc,
2000 str(parsed_url.path),
2001 postdata,
2002 files,
2003 parsed_url.scheme == 'https')
2004
2005 # Otherwise send a normal HTTP POST request
2006 else:
2007 encoded_postdata = None
2008 if postdata:
2009 encoded_postdata = urllib.urlencode(postdata)
2010 headers = [('Referer', self._referer),
2011 ('User-Agent', self._useragent)]
2012 if compress:
2013 headers.append(('Accept-encoding', 'gzip'))
2014 opener = urllib2.build_opener()
2015 opener.addheaders = headers
2016 resp = opener.open(url, data=encoded_postdata)
2017 if resp.info().get('Content-Encoding') == 'gzip':
2018 buf = StringIO(resp.read())
2019 f = gzip.GzipFile(fileobj=buf)
2020 resp_data = f.read()
2021 else:
2022 resp_data = resp.read()
2023
2024 # Parse the response into JSON
2025 if _log.isEnabledFor(logging.DEBUG):
2026 _log.debug('RESPONSE: ' + url + ', ' + _unicode_to_ascii(resp_data))
2027
2028 resp_json = json.loads(resp_data)
2029
2030 # Convert to ascii if directed to do so
2031 if self.ensure_ascii:
2032 resp_json = _unicode_to_ascii(resp_json)
2033
2034 # Check for errors, and handle the case where the token timed out
2035 # during use (and simply needs to be re-generated)
2036 try:
2037 if resp_json.get('error', None):
2038 errorcode = resp_json['error']['code']
2039 if errorcode == 498 and not is_retry:
2040 _log.info('Token expired during post request, fetching a new '
2041 + 'token and retrying')
2042 self.logout()
2043 newtoken = self.relogin()
2044 postdata['token'] = newtoken
2045 return self.post(path, postdata, files, ssl, compress,
2046 is_retry=True)
2047 elif errorcode == 498:
2048 raise RuntimeError('Invalid token')
2049 self._handle_json_error(resp_json['error'])
2050 return None
2051 except AttributeError:
2052 # Top-level JSON object isnt a dict, so can't have an error
2053 pass
2054
2055 return resp_json
2056
2057 def _postmultipart(self, host, selector, fields, files, ssl):
2058 boundary, body = self._encode_multipart_formdata(fields, files)
2059 headers = {
2060 'User-Agent': self._useragent,
2061 'Referer': self._referer,
2062 'Content-Type': 'multipart/form-data; boundary=%s' % boundary
2063 }
2064 if self.proxy_host:
2065 if ssl:
2066 h = httplib.HTTPSConnection(self.proxy_host, self.proxy_port,
2067 key_file=self.key_file,
2068 cert_file=self.cert_file)
2069 h.request('POST', 'https://' + host + selector, body, headers)
2070 else:
2071 h = httplib.HTTPConnection(self.proxy_host, self.proxy_port)
2072 h.request('POST', 'http://' + host + selector, body, headers)
2073 else:
2074 if ssl:
2075 h = httplib.HTTPSConnection(host, key_file=self.key_file,
2076 cert_file=self.cert_file)
2077 h.request('POST', selector, body, headers)
2078 else:
2079 h = httplib.HTTPConnection(host)
2080 h.request('POST', selector, body, headers)
2081 return h.getresponse().read()
2082
2083 def _encode_multipart_formdata(self, fields, files):
2084 boundary = mimetools.choose_boundary()
2085 buf = StringIO()
2086 for (key, value) in fields.iteritems():
2087 buf.write('--%s\r\n' % boundary)
2088 buf.write('Content-Disposition: form-data; name="%s"' % key)
2089 buf.write('\r\n\r\n' + _tostr(value) + '\r\n')
2090 for (key, filepath, filename) in files:
2091 buf.write('--%s\r\n' % boundary)
2092 buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
2093 buf.write('Content-Type: %s\r\n' % (self._get_content_type(filename)))
2094 f = open(filepath, "rb")
2095 try:
2096 buf.write('\r\n' + f.read() + '\r\n')
2097 finally:
2098 f.close()
2099 buf.write('--' + boundary + '--\r\n\r\n')
2100 buf = buf.getvalue()
2101 return boundary, buf
2102
2103 def _get_content_type(self, filename):
2104 return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
2105
2106 def _handle_json_error(self, error):
2107 _log.error(error.get('message', 'Unknown Error'))
2108 for errordetail in error['details']:
2109 _log.error(errordetail)
2110
2111
2112class _StrictURLopener(urllib.FancyURLopener):
2113 def http_error_default(self, url, fp, errcode, errmsg, headers):
2114 if errcode != 200:
2115 raise urllib2.HTTPError(url, errcode, errmsg, headers, fp)
2116
2117def _normalize_url(url, charset='utf-8'):
2118 """ Normalizes a URL. Based on http://code.google.com/p/url-normalize."""
2119 def _clean(string):
2120 string = unicode(urllib.unquote(string), 'utf-8', 'replace')
2121 return unicodedata.normalize('NFC', string).encode('utf-8')
2122
2123 default_port = {
2124 'ftp': 21,
2125 'telnet': 23,
2126 'http': 80,
2127 'gopher': 70,
2128 'news': 119,
2129 'nntp': 119,
2130 'prospero': 191,
2131 'https': 443,
2132 'snews': 563,
2133 'snntp': 563,
2134 }
2135 if isinstance(url, unicode):
2136 url = url.encode(charset, 'ignore')
2137
2138 # if there is no scheme use http as default scheme
2139 if url[0] not in ['/', '-'] and ':' not in url[:7]:
2140 url = 'http://' + url
2141
2142 # shebang urls support
2143 url = url.replace('#!', '?_escaped_fragment_=')
2144
2145 # splitting url to useful parts
2146 scheme, auth, path, query, fragment = urlparse.urlsplit(url.strip())
2147 (userinfo, host, port) = re.search('([^@]*@)?([^:]*):?(.*)', auth).groups()
2148
2149 # Always provide the URI scheme in lowercase characters.
2150 scheme = scheme.lower()
2151
2152 # Always provide the host, if any, in lowercase characters.
2153 host = host.lower()
2154 if host and host[-1] == '.':
2155 host = host[:-1]
2156 # take care about IDN domains
2157 host = host.decode(charset).encode('idna') # IDN -> ACE
2158
2159 # Only perform percent-encoding where it is essential.
2160 # Always use uppercase A-through-F characters when percent-encoding.
2161 # All portions of the URI must be utf-8 encoded NFC from Unicode strings
2162 path = urllib.quote(_clean(path), "~:/?#[]@!$&'()*+,;=")
2163 fragment = urllib.quote(_clean(fragment), "~")
2164
2165 # note care must be taken to only encode & and = characters as values
2166 query = "&".join(["=".join([urllib.quote(_clean(t), "~:/?#[]@!$'()*+,;=") \
2167 for t in q.split("=", 1)]) for q in query.split("&")])
2168
2169 # Prevent dot-segments appearing in non-relative URI paths.
2170 if scheme in ["", "http", "https", "ftp", "file"]:
2171 output = []
2172 for part in path.split('/'):
2173 if part == "":
2174 if not output:
2175 output.append(part)
2176 elif part == ".":
2177 pass
2178 elif part == "..":
2179 if len(output) > 1:
2180 output.pop()
2181 else:
2182 output.append(part)
2183 if part in ["", ".", ".."]:
2184 output.append("")
2185 path = '/'.join(output)
2186
2187 # For schemes that define a default authority, use an empty authority if
2188 # the default is desired.
2189 if userinfo in ["@", ":@"]:
2190 userinfo = ""
2191
2192 # For schemes that define an empty path to be equivalent to a path of "/",
2193 # use "/".
2194 if path == "" and scheme in ["http", "https", "ftp", "file"]:
2195 path = "/"
2196
2197 # For schemes that define a port, use an empty port if the default is
2198 # desired
2199 if port and scheme in default_port.keys():
2200 if port.isdigit():
2201 port = str(int(port))
2202 if int(port) == default_port[scheme]:
2203 port = ''
2204
2205 # Put it all back together again
2206 auth = (userinfo or "") + host
2207 if port:
2208 auth += ":" + port
2209 if url.endswith("#") and query == "" and fragment == "":
2210 path += "#"
2211 return urlparse.urlunsplit((scheme, auth, path, query, fragment))
2212
2213def _parse_hostname(url, include_port=False):
2214 """ Parses the hostname out of a URL."""
2215 if url:
2216 parsed_url = urlparse.urlparse((url))
2217 return parsed_url.netloc if include_port else parsed_url.hostname
2218
2219def _is_http_url(url):
2220 if url:
2221 return urlparse.urlparse(url).scheme in ['http', 'https']
2222
2223def _unpack(obj_or_seq, key=None, flatten=False):
2224 """ Turns a list of single item dicts in a list of the dict's values."""
2225
2226 # The trivial case (passed in None, return None)
2227 if not obj_or_seq:
2228 return None
2229
2230 # We assume it's a sequence
2231 new_list = []
2232 for obj in obj_or_seq:
2233 value = _unpack_obj(obj, key, flatten)
2234 new_list.extend(value)
2235
2236 return new_list
2237
2238def _unpack_obj(obj, key=None, flatten=False):
2239 try:
2240 if key:
2241 value = [obj.get(key)]
2242 else:
2243 value = obj.values()
2244 except AttributeError:
2245 value = [obj]
2246
2247 # Flatten any lists if directed to do so
2248 if value and flatten:
2249 value = [item for sublist in value for item in sublist]
2250
2251 return value
2252
2253def _unicode_to_ascii(data):
2254 """ Converts strings and collections of strings from unicode to ascii. """
2255 if isinstance(data, str):
2256 return _remove_non_ascii(data)
2257 if isinstance(data, unicode):
2258 return _remove_non_ascii(str(data.encode('utf8')))
2259 elif isinstance(data, collections.Mapping):
2260 return dict(map(_unicode_to_ascii, data.iteritems()))
2261 elif isinstance(data, collections.Iterable):
2262 return type(data)(map(_unicode_to_ascii, data))
2263 else:
2264 return data
2265
2266def _remove_non_ascii(s):
2267 return ''.join(i for i in s if ord(i) < 128)
2268
2269def _tostr(obj):
2270 if not obj:
2271 return ''
2272 if isinstance(obj, list):
2273 return ', '.join(map(_tostr, obj))
2274 return str(obj)
2275
2276
2277
2278
2279# This function is a workaround to deal with what's typically described as a
2280# problem with the web server closing a connection. This is problem
2281# experienced with www.arcgis.com (first encountered 12/13/2012). The problem
2282# and workaround is described here:
2283# http://bobrochel.blogspot.com/2010/11/bad-servers-chunked-encoding-and.html
2284def _patch_http_response_read(func):
2285 def inner(*args):
2286 try:
2287 return func(*args)
2288 except httplib.IncompleteRead, e:
2289 return e.partial
2290
2291 return inner
2292httplib.HTTPResponse.read = _patch_http_response_read(httplib.HTTPResponse.read)