· 6 years ago · Dec 02, 2019, 05:08 AM
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# Copyright 2012 Matt Martz
4# All Rights Reserved.
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
18import os
19import re
20import csv
21import sys
22import math
23import errno
24import signal
25import socket
26import timeit
27import datetime
28import platform
29import threading
30import xml.parsers.expat
31
32try:
33 import gzip
34 GZIP_BASE = gzip.GzipFile
35except ImportError:
36 gzip = None
37 GZIP_BASE = object
38
39__version__ = '2.1.2'
40
41
42class FakeShutdownEvent(object):
43 """Class to fake a threading.Event.isSet so that users of this module
44 are not required to register their own threading.Event()
45 """
46
47 @staticmethod
48 def isSet():
49 "Dummy method to always return false"""
50 return False
51
52
53# Some global variables we use
54DEBUG = False
55_GLOBAL_DEFAULT_TIMEOUT = object()
56PY25PLUS = sys.version_info[:2] >= (2, 5)
57PY26PLUS = sys.version_info[:2] >= (2, 6)
58PY32PLUS = sys.version_info[:2] >= (3, 2)
59
60# Begin import game to handle Python 2 and Python 3
61try:
62 import json
63except ImportError:
64 try:
65 import simplejson as json
66 except ImportError:
67 json = None
68
69try:
70 import xml.etree.ElementTree as ET
71 try:
72 from xml.etree.ElementTree import _Element as ET_Element
73 except ImportError:
74 pass
75except ImportError:
76 from xml.dom import minidom as DOM
77 from xml.parsers.expat import ExpatError
78 ET = None
79
80try:
81 from urllib2 import (urlopen, Request, HTTPError, URLError,
82 AbstractHTTPHandler, ProxyHandler,
83 HTTPDefaultErrorHandler, HTTPRedirectHandler,
84 HTTPErrorProcessor, OpenerDirector)
85except ImportError:
86 from urllib.request import (urlopen, Request, HTTPError, URLError,
87 AbstractHTTPHandler, ProxyHandler,
88 HTTPDefaultErrorHandler, HTTPRedirectHandler,
89 HTTPErrorProcessor, OpenerDirector)
90
91try:
92 from httplib import HTTPConnection, BadStatusLine
93except ImportError:
94 from http.client import HTTPConnection, BadStatusLine
95
96try:
97 from httplib import HTTPSConnection
98except ImportError:
99 try:
100 from http.client import HTTPSConnection
101 except ImportError:
102 HTTPSConnection = None
103
104try:
105 from httplib import FakeSocket
106except ImportError:
107 FakeSocket = None
108
109try:
110 from Queue import Queue
111except ImportError:
112 from queue import Queue
113
114try:
115 from urlparse import urlparse
116except ImportError:
117 from urllib.parse import urlparse
118
119try:
120 from urlparse import parse_qs
121except ImportError:
122 try:
123 from urllib.parse import parse_qs
124 except ImportError:
125 from cgi import parse_qs
126
127try:
128 from hashlib import md5
129except ImportError:
130 from md5 import md5
131
132try:
133 from argparse import ArgumentParser as ArgParser
134 from argparse import SUPPRESS as ARG_SUPPRESS
135 PARSER_TYPE_INT = int
136 PARSER_TYPE_STR = str
137 PARSER_TYPE_FLOAT = float
138except ImportError:
139 from optparse import OptionParser as ArgParser
140 from optparse import SUPPRESS_HELP as ARG_SUPPRESS
141 PARSER_TYPE_INT = 'int'
142 PARSER_TYPE_STR = 'string'
143 PARSER_TYPE_FLOAT = 'float'
144
145try:
146 from cStringIO import StringIO
147 BytesIO = None
148except ImportError:
149 try:
150 from StringIO import StringIO
151 BytesIO = None
152 except ImportError:
153 from io import StringIO, BytesIO
154
155try:
156 import __builtin__
157except ImportError:
158 import builtins
159 from io import TextIOWrapper, FileIO
160
161 class _Py3Utf8Output(TextIOWrapper):
162 """UTF-8 encoded wrapper around stdout for py3, to override
163 ASCII stdout
164 """
165 def __init__(self, f, **kwargs):
166 buf = FileIO(f.fileno(), 'w')
167 super(_Py3Utf8Output, self).__init__(
168 buf,
169 encoding='utf8',
170 errors='strict'
171 )
172
173 def write(self, s):
174 super(_Py3Utf8Output, self).write(s)
175 self.flush()
176
177 _py3_print = getattr(builtins, 'print')
178 try:
179 _py3_utf8_stdout = _Py3Utf8Output(sys.stdout)
180 _py3_utf8_stderr = _Py3Utf8Output(sys.stderr)
181 except OSError:
182 # sys.stdout/sys.stderr is not a compatible stdout/stderr object
183 # just use it and hope things go ok
184 _py3_utf8_stdout = sys.stdout
185 _py3_utf8_stderr = sys.stderr
186
187 def to_utf8(v):
188 """No-op encode to utf-8 for py3"""
189 return v
190
191 def print_(*args, **kwargs):
192 """Wrapper function for py3 to print, with a utf-8 encoded stdout"""
193 if kwargs.get('file') == sys.stderr:
194 kwargs['file'] = _py3_utf8_stderr
195 else:
196 kwargs['file'] = kwargs.get('file', _py3_utf8_stdout)
197 _py3_print(*args, **kwargs)
198else:
199 del __builtin__
200
201 def to_utf8(v):
202 """Encode value to utf-8 if possible for py2"""
203 try:
204 return v.encode('utf8', 'strict')
205 except AttributeError:
206 return v
207
208 def print_(*args, **kwargs):
209 """The new-style print function for Python 2.4 and 2.5.
210 Taken from https://pypi.python.org/pypi/six/
211 Modified to set encoding to UTF-8 always, and to flush after write
212 """
213 fp = kwargs.pop("file", sys.stdout)
214 if fp is None:
215 return
216
217 def write(data):
218 if not isinstance(data, basestring):
219 data = str(data)
220 # If the file has an encoding, encode unicode with it.
221 encoding = 'utf8' # Always trust UTF-8 for output
222 if (isinstance(fp, file) and
223 isinstance(data, unicode) and
224 encoding is not None):
225 errors = getattr(fp, "errors", None)
226 if errors is None:
227 errors = "strict"
228 data = data.encode(encoding, errors)
229 fp.write(data)
230 fp.flush()
231 want_unicode = False
232 sep = kwargs.pop("sep", None)
233 if sep is not None:
234 if isinstance(sep, unicode):
235 want_unicode = True
236 elif not isinstance(sep, str):
237 raise TypeError("sep must be None or a string")
238 end = kwargs.pop("end", None)
239 if end is not None:
240 if isinstance(end, unicode):
241 want_unicode = True
242 elif not isinstance(end, str):
243 raise TypeError("end must be None or a string")
244 if kwargs:
245 raise TypeError("invalid keyword arguments to print()")
246 if not want_unicode:
247 for arg in args:
248 if isinstance(arg, unicode):
249 want_unicode = True
250 break
251 if want_unicode:
252 newline = unicode("\n")
253 space = unicode(" ")
254 else:
255 newline = "\n"
256 space = " "
257 if sep is None:
258 sep = space
259 if end is None:
260 end = newline
261 for i, arg in enumerate(args):
262 if i:
263 write(sep)
264 write(arg)
265 write(end)
266
267if PY32PLUS:
268 etree_iter = ET.Element.iter
269elif PY25PLUS:
270 etree_iter = ET_Element.getiterator
271
272if PY26PLUS:
273 thread_is_alive = threading.Thread.is_alive
274else:
275 thread_is_alive = threading.Thread.isAlive
276
277
278# Exception "constants" to support Python 2 through Python 3
279try:
280 import ssl
281 try:
282 CERT_ERROR = (ssl.CertificateError,)
283 except AttributeError:
284 CERT_ERROR = tuple()
285
286 HTTP_ERRORS = (
287 (HTTPError, URLError, socket.error, ssl.SSLError, BadStatusLine) +
288 CERT_ERROR
289 )
290except ImportError:
291 ssl = None
292 HTTP_ERRORS = (HTTPError, URLError, socket.error, BadStatusLine)
293
294
295class SpeedtestException(Exception):
296 """Base exception for this module"""
297
298
299class SpeedtestCLIError(SpeedtestException):
300 """Generic exception for raising errors during CLI operation"""
301
302
303class SpeedtestHTTPError(SpeedtestException):
304 """Base HTTP exception for this module"""
305
306
307class SpeedtestConfigError(SpeedtestException):
308 """Configuration XML is invalid"""
309
310
311class SpeedtestServersError(SpeedtestException):
312 """Servers XML is invalid"""
313
314
315class ConfigRetrievalError(SpeedtestHTTPError):
316 """Could not retrieve config.php"""
317
318
319class ServersRetrievalError(SpeedtestHTTPError):
320 """Could not retrieve speedtest-servers.php"""
321
322
323class InvalidServerIDType(SpeedtestException):
324 """Server ID used for filtering was not an integer"""
325
326
327class NoMatchedServers(SpeedtestException):
328 """No servers matched when filtering"""
329
330
331class SpeedtestMiniConnectFailure(SpeedtestException):
332 """Could not connect to the provided speedtest mini server"""
333
334
335class InvalidSpeedtestMiniServer(SpeedtestException):
336 """Server provided as a speedtest mini server does not actually appear
337 to be a speedtest mini server
338 """
339
340
341class ShareResultsConnectFailure(SpeedtestException):
342 """Could not connect to speedtest.net API to POST results"""
343
344
345class ShareResultsSubmitFailure(SpeedtestException):
346 """Unable to successfully POST results to speedtest.net API after
347 connection
348 """
349
350
351class SpeedtestUploadTimeout(SpeedtestException):
352 """testlength configuration reached during upload
353 Used to ensure the upload halts when no additional data should be sent
354 """
355
356
357class SpeedtestBestServerFailure(SpeedtestException):
358 """Unable to determine best server"""
359
360
361class SpeedtestMissingBestServer(SpeedtestException):
362 """get_best_server not called or not able to determine best server"""
363
364
365def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
366 source_address=None):
367 """Connect to *address* and return the socket object.
368 Convenience function. Connect to *address* (a 2-tuple ``(host,
369 port)``) and return the socket object. Passing the optional
370 *timeout* parameter will set the timeout on the socket instance
371 before attempting to connect. If no *timeout* is supplied, the
372 global default timeout setting returned by :func:`getdefaulttimeout`
373 is used. If *source_address* is set it must be a tuple of (host, port)
374 for the socket to bind as a source address before making the connection.
375 An host of '' or port 0 tells the OS to use the default.
376 Largely vendored from Python 2.7, modified to work with Python 2.4
377 """
378
379 host, port = address
380 err = None
381 for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
382 af, socktype, proto, canonname, sa = res
383 sock = None
384 try:
385 sock = socket.socket(af, socktype, proto)
386 if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
387 sock.settimeout(float(timeout))
388 if source_address:
389 sock.bind(source_address)
390 sock.connect(sa)
391 return sock
392
393 except socket.error:
394 err = get_exception()
395 if sock is not None:
396 sock.close()
397
398 if err is not None:
399 raise err
400 else:
401 raise socket.error("getaddrinfo returns an empty list")
402
403
404class SpeedtestHTTPConnection(HTTPConnection):
405 """Custom HTTPConnection to support source_address across
406 Python 2.4 - Python 3
407 """
408 def __init__(self, *args, **kwargs):
409 source_address = kwargs.pop('source_address', None)
410 timeout = kwargs.pop('timeout', 10)
411
412 self._tunnel_host = None
413
414 HTTPConnection.__init__(self, *args, **kwargs)
415
416 self.source_address = source_address
417 self.timeout = timeout
418
419 def connect(self):
420 """Connect to the host and port specified in __init__."""
421 try:
422 self.sock = socket.create_connection(
423 (self.host, self.port),
424 self.timeout,
425 self.source_address
426 )
427 except (AttributeError, TypeError):
428 self.sock = create_connection(
429 (self.host, self.port),
430 self.timeout,
431 self.source_address
432 )
433
434 if self._tunnel_host:
435 self._tunnel()
436
437
438if HTTPSConnection:
439 class SpeedtestHTTPSConnection(HTTPSConnection):
440 """Custom HTTPSConnection to support source_address across
441 Python 2.4 - Python 3
442 """
443 default_port = 443
444
445 def __init__(self, *args, **kwargs):
446 source_address = kwargs.pop('source_address', None)
447 timeout = kwargs.pop('timeout', 10)
448
449 self._tunnel_host = None
450
451 HTTPSConnection.__init__(self, *args, **kwargs)
452
453 self.timeout = timeout
454 self.source_address = source_address
455
456 def connect(self):
457 "Connect to a host on a given (SSL) port."
458 try:
459 self.sock = socket.create_connection(
460 (self.host, self.port),
461 self.timeout,
462 self.source_address
463 )
464 except (AttributeError, TypeError):
465 self.sock = create_connection(
466 (self.host, self.port),
467 self.timeout,
468 self.source_address
469 )
470
471 if self._tunnel_host:
472 self._tunnel()
473
474 if ssl:
475 try:
476 kwargs = {}
477 if hasattr(ssl, 'SSLContext'):
478 if self._tunnel_host:
479 kwargs['server_hostname'] = self._tunnel_host
480 else:
481 kwargs['server_hostname'] = self.host
482 self.sock = self._context.wrap_socket(self.sock, **kwargs)
483 except AttributeError:
484 self.sock = ssl.wrap_socket(self.sock)
485 try:
486 self.sock.server_hostname = self.host
487 except AttributeError:
488 pass
489 elif FakeSocket:
490 # Python 2.4/2.5 support
491 try:
492 self.sock = FakeSocket(self.sock, socket.ssl(self.sock))
493 except AttributeError:
494 raise SpeedtestException(
495 'This version of Python does not support HTTPS/SSL '
496 'functionality'
497 )
498 else:
499 raise SpeedtestException(
500 'This version of Python does not support HTTPS/SSL '
501 'functionality'
502 )
503
504
505def _build_connection(connection, source_address, timeout, context=None):
506 """Cross Python 2.4 - Python 3 callable to build an ``HTTPConnection`` or
507 ``HTTPSConnection`` with the args we need
508 Called from ``http(s)_open`` methods of ``SpeedtestHTTPHandler`` or
509 ``SpeedtestHTTPSHandler``
510 """
511 def inner(host, **kwargs):
512 kwargs.update({
513 'source_address': source_address,
514 'timeout': timeout
515 })
516 if context:
517 kwargs['context'] = context
518 return connection(host, **kwargs)
519 return inner
520
521
522class SpeedtestHTTPHandler(AbstractHTTPHandler):
523 """Custom ``HTTPHandler`` that can build a ``HTTPConnection`` with the
524 args we need for ``source_address`` and ``timeout``
525 """
526 def __init__(self, debuglevel=0, source_address=None, timeout=10):
527 AbstractHTTPHandler.__init__(self, debuglevel)
528 self.source_address = source_address
529 self.timeout = timeout
530
531 def http_open(self, req):
532 return self.do_open(
533 _build_connection(
534 SpeedtestHTTPConnection,
535 self.source_address,
536 self.timeout
537 ),
538 req
539 )
540
541 http_request = AbstractHTTPHandler.do_request_
542
543
544class SpeedtestHTTPSHandler(AbstractHTTPHandler):
545 """Custom ``HTTPSHandler`` that can build a ``HTTPSConnection`` with the
546 args we need for ``source_address`` and ``timeout``
547 """
548 def __init__(self, debuglevel=0, context=None, source_address=None,
549 timeout=10):
550 AbstractHTTPHandler.__init__(self, debuglevel)
551 self._context = context
552 self.source_address = source_address
553 self.timeout = timeout
554
555 def https_open(self, req):
556 return self.do_open(
557 _build_connection(
558 SpeedtestHTTPSConnection,
559 self.source_address,
560 self.timeout,
561 context=self._context,
562 ),
563 req
564 )
565
566 https_request = AbstractHTTPHandler.do_request_
567
568
569def build_opener(source_address=None, timeout=10):
570 """Function similar to ``urllib2.build_opener`` that will build
571 an ``OpenerDirector`` with the explicit handlers we want,
572 ``source_address`` for binding, ``timeout`` and our custom
573 `User-Agent`
574 """
575
576 printer('Timeout set to %d' % timeout, debug=True)
577
578 if source_address:
579 source_address_tuple = (source_address, 0)
580 printer('Binding to source address: %r' % (source_address_tuple,),
581 debug=True)
582 else:
583 source_address_tuple = None
584
585 handlers = [
586 ProxyHandler(),
587 SpeedtestHTTPHandler(source_address=source_address_tuple,
588 timeout=timeout),
589 SpeedtestHTTPSHandler(source_address=source_address_tuple,
590 timeout=timeout),
591 HTTPDefaultErrorHandler(),
592 HTTPRedirectHandler(),
593 HTTPErrorProcessor()
594 ]
595
596 opener = OpenerDirector()
597 opener.addheaders = [('User-agent', build_user_agent())]
598
599 for handler in handlers:
600 opener.add_handler(handler)
601
602 return opener
603
604
605class GzipDecodedResponse(GZIP_BASE):
606 """A file-like object to decode a response encoded with the gzip
607 method, as described in RFC 1952.
608 Largely copied from ``xmlrpclib``/``xmlrpc.client`` and modified
609 to work for py2.4-py3
610 """
611 def __init__(self, response):
612 # response doesn't support tell() and read(), required by
613 # GzipFile
614 if not gzip:
615 raise SpeedtestHTTPError('HTTP response body is gzip encoded, '
616 'but gzip support is not available')
617 IO = BytesIO or StringIO
618 self.io = IO()
619 while 1:
620 chunk = response.read(1024)
621 if len(chunk) == 0:
622 break
623 self.io.write(chunk)
624 self.io.seek(0)
625 gzip.GzipFile.__init__(self, mode='rb', fileobj=self.io)
626
627 def close(self):
628 try:
629 gzip.GzipFile.close(self)
630 finally:
631 self.io.close()
632
633
634def get_exception():
635 """Helper function to work with py2.4-py3 for getting the current
636 exception in a try/except block
637 """
638 return sys.exc_info()[1]
639
640
641def distance(origin, destination):
642 """Determine distance between 2 sets of [lat,lon] in km"""
643
644 lat1, lon1 = origin
645 lat2, lon2 = destination
646 radius = 6371 # km
647
648 dlat = math.radians(lat2 - lat1)
649 dlon = math.radians(lon2 - lon1)
650 a = (math.sin(dlat / 2) * math.sin(dlat / 2) +
651 math.cos(math.radians(lat1)) *
652 math.cos(math.radians(lat2)) * math.sin(dlon / 2) *
653 math.sin(dlon / 2))
654 c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
655 d = radius * c
656
657 return d
658
659
660def build_user_agent():
661 """Build a Mozilla/5.0 compatible User-Agent string"""
662
663 ua_tuple = (
664 'Mozilla/5.0',
665 '(%s; U; %s; en-us)' % (platform.platform(),
666 platform.architecture()[0]),
667 'Python/%s' % platform.python_version(),
668 '(KHTML, like Gecko)',
669 'speedtest-cli/%s' % __version__
670 )
671 user_agent = ' '.join(ua_tuple)
672 printer('User-Agent: %s' % user_agent, debug=True)
673 return user_agent
674
675
676def build_request(url, data=None, headers=None, bump='0', secure=False):
677 """Build a urllib2 request object
678 This function automatically adds a User-Agent header to all requests
679 """
680
681 if not headers:
682 headers = {}
683
684 if url[0] == ':':
685 scheme = ('http', 'https')[bool(secure)]
686 schemed_url = '%s%s' % (scheme, url)
687 else:
688 schemed_url = url
689
690 if '?' in url:
691 delim = '&'
692 else:
693 delim = '?'
694
695 # WHO YOU GONNA CALL? CACHE BUSTERS!
696 final_url = '%s%sx=%s.%s' % (schemed_url, delim,
697 int(timeit.time.time() * 1000),
698 bump)
699
700 headers.update({
701 'Cache-Control': 'no-cache',
702 })
703
704 printer('%s %s' % (('GET', 'POST')[bool(data)], final_url),
705 debug=True)
706
707 return Request(final_url, data=data, headers=headers)
708
709
710def catch_request(request, opener=None):
711 """Helper function to catch common exceptions encountered when
712 establishing a connection with a HTTP/HTTPS request
713 """
714
715 if opener:
716 _open = opener.open
717 else:
718 _open = urlopen
719
720 try:
721 uh = _open(request)
722 if request.get_full_url() != uh.geturl():
723 printer('Redirected to %s' % uh.geturl(), debug=True)
724 return uh, False
725 except HTTP_ERRORS:
726 e = get_exception()
727 return None, e
728
729
730def get_response_stream(response):
731 """Helper function to return either a Gzip reader if
732 ``Content-Encoding`` is ``gzip`` otherwise the response itself
733 """
734
735 try:
736 getheader = response.headers.getheader
737 except AttributeError:
738 getheader = response.getheader
739
740 if getheader('content-encoding') == 'gzip':
741 return GzipDecodedResponse(response)
742
743 return response
744
745
746def get_attributes_by_tag_name(dom, tag_name):
747 """Retrieve an attribute from an XML document and return it in a
748 consistent format
749 Only used with xml.dom.minidom, which is likely only to be used
750 with python versions older than 2.5
751 """
752 elem = dom.getElementsByTagName(tag_name)[0]
753 return dict(list(elem.attributes.items()))
754
755
756def print_dots(shutdown_event):
757 """Built in callback function used by Thread classes for printing
758 status
759 """
760 def inner(current, total, start=False, end=False):
761 if shutdown_event.isSet():
762 return
763
764 sys.stdout.write('.')
765 if current + 1 == total and end is True:
766 sys.stdout.write('\n')
767 sys.stdout.flush()
768 return inner
769
770
771def do_nothing(*args, **kwargs):
772 pass
773
774
775class HTTPDownloader(threading.Thread):
776 """Thread class for retrieving a URL"""
777
778 def __init__(self, i, request, start, timeout, opener=None,
779 shutdown_event=None):
780 threading.Thread.__init__(self)
781 self.request = request
782 self.result = [0]
783 self.starttime = start
784 self.timeout = timeout
785 self.i = i
786 if opener:
787 self._opener = opener.open
788 else:
789 self._opener = urlopen
790
791 if shutdown_event:
792 self._shutdown_event = shutdown_event
793 else:
794 self._shutdown_event = FakeShutdownEvent()
795
796 def run(self):
797 try:
798 if (timeit.default_timer() - self.starttime) <= self.timeout:
799 f = self._opener(self.request)
800 while (not self._shutdown_event.isSet() and
801 (timeit.default_timer() - self.starttime) <=
802 self.timeout):
803 self.result.append(len(f.read(10240)))
804 if self.result[-1] == 0:
805 break
806 f.close()
807 except IOError:
808 pass
809
810
811class HTTPUploaderData(object):
812 """File like object to improve cutting off the upload once the timeout
813 has been reached
814 """
815
816 def __init__(self, length, start, timeout, shutdown_event=None):
817 self.length = length
818 self.start = start
819 self.timeout = timeout
820
821 if shutdown_event:
822 self._shutdown_event = shutdown_event
823 else:
824 self._shutdown_event = FakeShutdownEvent()
825
826 self._data = None
827
828 self.total = [0]
829
830 def pre_allocate(self):
831 chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'
832 multiplier = int(round(int(self.length) / 36.0))
833 IO = BytesIO or StringIO
834 try:
835 self._data = IO(
836 ('content1=%s' %
837 (chars * multiplier)[0:int(self.length) - 9]
838 ).encode()
839 )
840 except MemoryError:
841 raise SpeedtestCLIError(
842 'Insufficient memory to pre-allocate upload data. Please '
843 'use --no-pre-allocate'
844 )
845
846 @property
847 def data(self):
848 if not self._data:
849 self.pre_allocate()
850 return self._data
851
852 def read(self, n=10240):
853 if ((timeit.default_timer() - self.start) <= self.timeout and
854 not self._shutdown_event.isSet()):
855 chunk = self.data.read(n)
856 self.total.append(len(chunk))
857 return chunk
858 else:
859 raise SpeedtestUploadTimeout()
860
861 def __len__(self):
862 return self.length
863
864
865class HTTPUploader(threading.Thread):
866 """Thread class for putting a URL"""
867
868 def __init__(self, i, request, start, size, timeout, opener=None,
869 shutdown_event=None):
870 threading.Thread.__init__(self)
871 self.request = request
872 self.request.data.start = self.starttime = start
873 self.size = size
874 self.result = None
875 self.timeout = timeout
876 self.i = i
877
878 if opener:
879 self._opener = opener.open
880 else:
881 self._opener = urlopen
882
883 if shutdown_event:
884 self._shutdown_event = shutdown_event
885 else:
886 self._shutdown_event = FakeShutdownEvent()
887
888 def run(self):
889 request = self.request
890 try:
891 if ((timeit.default_timer() - self.starttime) <= self.timeout and
892 not self._shutdown_event.isSet()):
893 try:
894 f = self._opener(request)
895 except TypeError:
896 # PY24 expects a string or buffer
897 # This also causes issues with Ctrl-C, but we will concede
898 # for the moment that Ctrl-C on PY24 isn't immediate
899 request = build_request(self.request.get_full_url(),
900 data=request.data.read(self.size))
901 f = self._opener(request)
902 f.read(11)
903 f.close()
904 self.result = sum(self.request.data.total)
905 else:
906 self.result = 0
907 except (IOError, SpeedtestUploadTimeout):
908 self.result = sum(self.request.data.total)
909
910
911class SpeedtestResults(object):
912 """Class for holding the results of a speedtest, including:
913 Download speed
914 Upload speed
915 Ping/Latency to test server
916 Data about server that the test was run against
917 Additionally this class can return a result data as a dictionary or CSV,
918 as well as submit a POST of the result data to the speedtest.net API
919 to get a share results image link.
920 """
921
922 def __init__(self, download=0, upload=0, ping=0, server=None, client=None,
923 opener=None, secure=False):
924 self.download = download
925 self.upload = upload
926 self.ping = ping
927 if server is None:
928 self.server = {}
929 else:
930 self.server = server
931 self.client = client or {}
932
933 self._share = None
934 self.timestamp = '%sZ' % datetime.datetime.utcnow().isoformat()
935 self.bytes_received = 0
936 self.bytes_sent = 0
937
938 if opener:
939 self._opener = opener
940 else:
941 self._opener = build_opener()
942
943 self._secure = secure
944
945 def __repr__(self):
946 return repr(self.dict())
947
948 def share(self):
949 """POST data to the speedtest.net API to obtain a share results
950 link
951 """
952
953 if self._share:
954 return self._share
955
956 download = int(round(self.download / 1000.0, 0))
957 ping = int(round(self.ping, 0))
958 upload = int(round(self.upload / 1000.0, 0))
959
960 # Build the request to send results back to speedtest.net
961 # We use a list instead of a dict because the API expects parameters
962 # in a certain order
963 api_data = [
964 'recommendedserverid=%s' % self.server['id'],
965 'ping=%s' % ping,
966 'screenresolution=',
967 'promo=',
968 'download=%s' % download,
969 'screendpi=',
970 'upload=%s' % upload,
971 'testmethod=http',
972 'hash=%s' % md5(('%s-%s-%s-%s' %
973 (ping, upload, download, '297aae72'))
974 .encode()).hexdigest(),
975 'touchscreen=none',
976 'startmode=pingselect',
977 'accuracy=1',
978 'bytesreceived=%s' % self.bytes_received,
979 'bytessent=%s' % self.bytes_sent,
980 'serverid=%s' % self.server['id'],
981 ]
982
983 headers = {'Referer': 'http://c.speedtest.net/flash/speedtest.swf'}
984 request = build_request('://www.speedtest.net/api/api.php',
985 data='&'.join(api_data).encode(),
986 headers=headers, secure=self._secure)
987 f, e = catch_request(request, opener=self._opener)
988 if e:
989 raise ShareResultsConnectFailure(e)
990
991 response = f.read()
992 code = f.code
993 f.close()
994
995 if int(code) != 200:
996 raise ShareResultsSubmitFailure('Could not submit results to '
997 'speedtest.net')
998
999 qsargs = parse_qs(response.decode())
1000 resultid = qsargs.get('resultid')
1001 if not resultid or len(resultid) != 1:
1002 raise ShareResultsSubmitFailure('Could not submit results to '
1003 'speedtest.net')
1004
1005 self._share = 'http://www.speedtest.net/result/%s.png' % resultid[0]
1006
1007 return self._share
1008
1009 def dict(self):
1010 """Return dictionary of result data"""
1011
1012 return {
1013 'download': self.download,
1014 'upload': self.upload,
1015 'ping': self.ping,
1016 'server': self.server,
1017 'timestamp': self.timestamp,
1018 'bytes_sent': self.bytes_sent,
1019 'bytes_received': self.bytes_received,
1020 'share': self._share,
1021 'client': self.client,
1022 }
1023
1024 @staticmethod
1025 def csv_header(delimiter=','):
1026 """Return CSV Headers"""
1027
1028 row = ['Server ID', 'Sponsor', 'Server Name', 'Timestamp', 'Distance',
1029 'Ping', 'Download', 'Upload', 'Share', 'IP Address']
1030 out = StringIO()
1031 writer = csv.writer(out, delimiter=delimiter, lineterminator='')
1032 writer.writerow([to_utf8(v) for v in row])
1033 return out.getvalue()
1034
1035 def csv(self, delimiter=','):
1036 """Return data in CSV format"""
1037
1038 data = self.dict()
1039 out = StringIO()
1040 writer = csv.writer(out, delimiter=delimiter, lineterminator='')
1041 row = [data['server']['id'], data['server']['sponsor'],
1042 data['server']['name'], data['timestamp'],
1043 data['server']['d'], data['ping'], data['download'],
1044 data['upload'], self._share or '', self.client['ip']]
1045 writer.writerow([to_utf8(v) for v in row])
1046 return out.getvalue()
1047
1048 def json(self, pretty=False):
1049 """Return data in JSON format"""
1050
1051 kwargs = {}
1052 if pretty:
1053 kwargs.update({
1054 'indent': 4,
1055 'sort_keys': True
1056 })
1057 return json.dumps(self.dict(), **kwargs)
1058
1059
1060class Speedtest(object):
1061 """Class for performing standard speedtest.net testing operations"""
1062
1063 def __init__(self, config=None, source_address=None, timeout=10,
1064 secure=False, shutdown_event=None):
1065 self.config = {}
1066
1067 self._source_address = source_address
1068 self._timeout = timeout
1069 self._opener = build_opener(source_address, timeout)
1070
1071 self._secure = secure
1072
1073 if shutdown_event:
1074 self._shutdown_event = shutdown_event
1075 else:
1076 self._shutdown_event = FakeShutdownEvent()
1077
1078 self.get_config()
1079 if config is not None:
1080 self.config.update(config)
1081
1082 self.servers = {}
1083 self.closest = []
1084 self._best = {}
1085
1086 self.results = SpeedtestResults(
1087 client=self.config['client'],
1088 opener=self._opener,
1089 secure=secure,
1090 )
1091
1092 @property
1093 def best(self):
1094 if not self._best:
1095 self.get_best_server()
1096 return self._best
1097
1098 def get_config(self):
1099 """Download the speedtest.net configuration and return only the data
1100 we are interested in
1101 """
1102
1103 headers = {}
1104 if gzip:
1105 headers['Accept-Encoding'] = 'gzip'
1106 request = build_request('://www.speedtest.net/speedtest-config.php',
1107 headers=headers, secure=self._secure)
1108 uh, e = catch_request(request, opener=self._opener)
1109 if e:
1110 raise ConfigRetrievalError(e)
1111 configxml_list = []
1112
1113 stream = get_response_stream(uh)
1114
1115 while 1:
1116 try:
1117 configxml_list.append(stream.read(1024))
1118 except (OSError, EOFError):
1119 raise ConfigRetrievalError(get_exception())
1120 if len(configxml_list[-1]) == 0:
1121 break
1122 stream.close()
1123 uh.close()
1124
1125 if int(uh.code) != 200:
1126 return None
1127
1128 configxml = ''.encode().join(configxml_list)
1129
1130 printer('Config XML:\n%s' % configxml, debug=True)
1131
1132 try:
1133 try:
1134 root = ET.fromstring(configxml)
1135 except ET.ParseError:
1136 e = get_exception()
1137 raise SpeedtestConfigError(
1138 'Malformed speedtest.net configuration: %s' % e
1139 )
1140 server_config = root.find('server-config').attrib
1141 download = root.find('download').attrib
1142 upload = root.find('upload').attrib
1143 # times = root.find('times').attrib
1144 client = root.find('client').attrib
1145
1146 except AttributeError:
1147 try:
1148 root = DOM.parseString(configxml)
1149 except ExpatError:
1150 e = get_exception()
1151 raise SpeedtestConfigError(
1152 'Malformed speedtest.net configuration: %s' % e
1153 )
1154 server_config = get_attributes_by_tag_name(root, 'server-config')
1155 download = get_attributes_by_tag_name(root, 'download')
1156 upload = get_attributes_by_tag_name(root, 'upload')
1157 # times = get_attributes_by_tag_name(root, 'times')
1158 client = get_attributes_by_tag_name(root, 'client')
1159
1160 ignore_servers = list(
1161 map(int, server_config['ignoreids'].split(','))
1162 )
1163
1164 ratio = int(upload['ratio'])
1165 upload_max = int(upload['maxchunkcount'])
1166 up_sizes = [32768, 65536, 131072, 262144, 524288, 1048576, 7340032]
1167 sizes = {
1168 'upload': up_sizes[ratio - 1:],
1169 'download': [350, 500, 750, 1000, 1500, 2000, 2500,
1170 3000, 3500, 4000]
1171 }
1172
1173 size_count = len(sizes['upload'])
1174
1175 upload_count = int(math.ceil(upload_max / size_count))
1176
1177 counts = {
1178 'upload': upload_count,
1179 'download': int(download['threadsperurl'])
1180 }
1181
1182 threads = {
1183 'upload': int(upload['threads']),
1184 'download': int(server_config['threadcount']) * 2
1185 }
1186
1187 length = {
1188 'upload': int(upload['testlength']),
1189 'download': int(download['testlength'])
1190 }
1191
1192 self.config.update({
1193 'client': client,
1194 'ignore_servers': ignore_servers,
1195 'sizes': sizes,
1196 'counts': counts,
1197 'threads': threads,
1198 'length': length,
1199 'upload_max': upload_count * size_count
1200 })
1201
1202 try:
1203 self.lat_lon = (float(client['lat']), float(client['lon']))
1204 except ValueError:
1205 raise SpeedtestConfigError(
1206 'Unknown location: lat=%r lon=%r' %
1207 (client.get('lat'), client.get('lon'))
1208 )
1209
1210 printer('Config:\n%r' % self.config, debug=True)
1211
1212 return self.config
1213
1214 def get_servers(self, servers=None, exclude=None):
1215 """Retrieve a the list of speedtest.net servers, optionally filtered
1216 to servers matching those specified in the ``servers`` argument
1217 """
1218 if servers is None:
1219 servers = []
1220
1221 if exclude is None:
1222 exclude = []
1223
1224 self.servers.clear()
1225
1226 for server_list in (servers, exclude):
1227 for i, s in enumerate(server_list):
1228 try:
1229 server_list[i] = int(s)
1230 except ValueError:
1231 raise InvalidServerIDType(
1232 '%s is an invalid server type, must be int' % s
1233 )
1234
1235 urls = [
1236 '://www.speedtest.net/speedtest-servers-static.php',
1237 'http://c.speedtest.net/speedtest-servers-static.php',
1238 '://www.speedtest.net/speedtest-servers.php',
1239 'http://c.speedtest.net/speedtest-servers.php',
1240 ]
1241
1242 headers = {}
1243 if gzip:
1244 headers['Accept-Encoding'] = 'gzip'
1245
1246 errors = []
1247 for url in urls:
1248 try:
1249 request = build_request(
1250 '%s?threads=%s' % (url,
1251 self.config['threads']['download']),
1252 headers=headers,
1253 secure=self._secure
1254 )
1255 uh, e = catch_request(request, opener=self._opener)
1256 if e:
1257 errors.append('%s' % e)
1258 raise ServersRetrievalError()
1259
1260 stream = get_response_stream(uh)
1261
1262 serversxml_list = []
1263 while 1:
1264 try:
1265 serversxml_list.append(stream.read(1024))
1266 except (OSError, EOFError):
1267 raise ServersRetrievalError(get_exception())
1268 if len(serversxml_list[-1]) == 0:
1269 break
1270
1271 stream.close()
1272 uh.close()
1273
1274 if int(uh.code) != 200:
1275 raise ServersRetrievalError()
1276
1277 serversxml = ''.encode().join(serversxml_list)
1278
1279 printer('Servers XML:\n%s' % serversxml, debug=True)
1280
1281 try:
1282 try:
1283 try:
1284 root = ET.fromstring(serversxml)
1285 except ET.ParseError:
1286 e = get_exception()
1287 raise SpeedtestServersError(
1288 'Malformed speedtest.net server list: %s' % e
1289 )
1290 elements = etree_iter(root, 'server')
1291 except AttributeError:
1292 try:
1293 root = DOM.parseString(serversxml)
1294 except ExpatError:
1295 e = get_exception()
1296 raise SpeedtestServersError(
1297 'Malformed speedtest.net server list: %s' % e
1298 )
1299 elements = root.getElementsByTagName('server')
1300 except (SyntaxError, xml.parsers.expat.ExpatError):
1301 raise ServersRetrievalError()
1302
1303 for server in elements:
1304 try:
1305 attrib = server.attrib
1306 except AttributeError:
1307 attrib = dict(list(server.attributes.items()))
1308
1309 if servers and int(attrib.get('id')) not in servers:
1310 continue
1311
1312 if (int(attrib.get('id')) in self.config['ignore_servers']
1313 or int(attrib.get('id')) in exclude):
1314 continue
1315
1316 try:
1317 d = distance(self.lat_lon,
1318 (float(attrib.get('lat')),
1319 float(attrib.get('lon'))))
1320 except Exception:
1321 continue
1322
1323 attrib['d'] = d
1324
1325 try:
1326 self.servers[d].append(attrib)
1327 except KeyError:
1328 self.servers[d] = [attrib]
1329
1330 break
1331
1332 except ServersRetrievalError:
1333 continue
1334
1335 if (servers or exclude) and not self.servers:
1336 raise NoMatchedServers()
1337
1338 return self.servers
1339
1340 def set_mini_server(self, server):
1341 """Instead of querying for a list of servers, set a link to a
1342 speedtest mini server
1343 """
1344
1345 urlparts = urlparse(server)
1346
1347 name, ext = os.path.splitext(urlparts[2])
1348 if ext:
1349 url = os.path.dirname(server)
1350 else:
1351 url = server
1352
1353 request = build_request(url)
1354 uh, e = catch_request(request, opener=self._opener)
1355 if e:
1356 raise SpeedtestMiniConnectFailure('Failed to connect to %s' %
1357 server)
1358 else:
1359 text = uh.read()
1360 uh.close()
1361
1362 extension = re.findall('upload_?[Ee]xtension: "([^"]+)"',
1363 text.decode())
1364 if not extension:
1365 for ext in ['php', 'asp', 'aspx', 'jsp']:
1366 try:
1367 f = self._opener.open(
1368 '%s/speedtest/upload.%s' % (url, ext)
1369 )
1370 except Exception:
1371 pass
1372 else:
1373 data = f.read().strip().decode()
1374 if (f.code == 200 and
1375 len(data.splitlines()) == 1 and
1376 re.match('size=[0-9]', data)):
1377 extension = [ext]
1378 break
1379 if not urlparts or not extension:
1380 raise InvalidSpeedtestMiniServer('Invalid Speedtest Mini Server: '
1381 '%s' % server)
1382
1383 self.servers = [{
1384 'sponsor': 'Speedtest Mini',
1385 'name': urlparts[1],
1386 'd': 0,
1387 'url': '%s/speedtest/upload.%s' % (url.rstrip('/'), extension[0]),
1388 'latency': 0,
1389 'id': 0
1390 }]
1391
1392 return self.servers
1393
1394 def get_closest_servers(self, limit=5):
1395 """Limit servers to the closest speedtest.net servers based on
1396 geographic distance
1397 """
1398
1399 if not self.servers:
1400 self.get_servers()
1401
1402 for d in sorted(self.servers.keys()):
1403 for s in self.servers[d]:
1404 self.closest.append(s)
1405 if len(self.closest) == limit:
1406 break
1407 else:
1408 continue
1409 break
1410
1411 printer('Closest Servers:\n%r' % self.closest, debug=True)
1412 return self.closest
1413
1414 def get_best_server(self, servers=None):
1415 """Perform a speedtest.net "ping" to determine which speedtest.net
1416 server has the lowest latency
1417 """
1418
1419 if not servers:
1420 if not self.closest:
1421 servers = self.get_closest_servers()
1422 servers = self.closest
1423
1424 if self._source_address:
1425 source_address_tuple = (self._source_address, 0)
1426 else:
1427 source_address_tuple = None
1428
1429 user_agent = build_user_agent()
1430
1431 results = {}
1432 for server in servers:
1433 cum = []
1434 url = os.path.dirname(server['url'])
1435 stamp = int(timeit.time.time() * 1000)
1436 latency_url = '%s/latency.txt?x=%s' % (url, stamp)
1437 for i in range(0, 3):
1438 this_latency_url = '%s.%s' % (latency_url, i)
1439 printer('%s %s' % ('GET', this_latency_url),
1440 debug=True)
1441 urlparts = urlparse(latency_url)
1442 try:
1443 if urlparts[0] == 'https':
1444 h = SpeedtestHTTPSConnection(
1445 urlparts[1],
1446 source_address=source_address_tuple
1447 )
1448 else:
1449 h = SpeedtestHTTPConnection(
1450 urlparts[1],
1451 source_address=source_address_tuple
1452 )
1453 headers = {'User-Agent': user_agent}
1454 path = '%s?%s' % (urlparts[2], urlparts[4])
1455 start = timeit.default_timer()
1456 h.request("GET", path, headers=headers)
1457 r = h.getresponse()
1458 total = (timeit.default_timer() - start)
1459 except HTTP_ERRORS:
1460 e = get_exception()
1461 printer('ERROR: %r' % e, debug=True)
1462 cum.append(3600)
1463 continue
1464
1465 text = r.read(9)
1466 if int(r.status) == 200 and text == 'test=test'.encode():
1467 cum.append(total)
1468 else:
1469 cum.append(3600)
1470 h.close()
1471
1472 avg = round((sum(cum) / 6) * 1000.0, 3)
1473 results[avg] = server
1474
1475 try:
1476 fastest = sorted(results.keys())[0]
1477 except IndexError:
1478 raise SpeedtestBestServerFailure('Unable to connect to servers to '
1479 'test latency.')
1480 best = results[fastest]
1481 best['latency'] = fastest
1482
1483 self.results.ping = fastest
1484 self.results.server = best
1485
1486 self._best.update(best)
1487 printer('Best Server:\n%r' % best, debug=True)
1488 return best
1489
1490 def download(self, callback=do_nothing, threads=None):
1491 """Test download speed against speedtest.net
1492 A ``threads`` value of ``None`` will fall back to those dictated
1493 by the speedtest.net configuration
1494 """
1495
1496 urls = []
1497 for size in self.config['sizes']['download']:
1498 for _ in range(0, self.config['counts']['download']):
1499 urls.append('%s/random%sx%s.jpg' %
1500 (os.path.dirname(self.best['url']), size, size))
1501
1502 request_count = len(urls)
1503 requests = []
1504 for i, url in enumerate(urls):
1505 requests.append(
1506 build_request(url, bump=i, secure=self._secure)
1507 )
1508
1509 max_threads = threads or self.config['threads']['download']
1510 in_flight = {'threads': 0}
1511
1512 def producer(q, requests, request_count):
1513 for i, request in enumerate(requests):
1514 thread = HTTPDownloader(
1515 i,
1516 request,
1517 start,
1518 self.config['length']['download'],
1519 opener=self._opener,
1520 shutdown_event=self._shutdown_event
1521 )
1522 while in_flight['threads'] >= max_threads:
1523 timeit.time.sleep(0.001)
1524 thread.start()
1525 q.put(thread, True)
1526 in_flight['threads'] += 1
1527 callback(i, request_count, start=True)
1528
1529 finished = []
1530
1531 def consumer(q, request_count):
1532 _is_alive = thread_is_alive
1533 while len(finished) < request_count:
1534 thread = q.get(True)
1535 while _is_alive(thread):
1536 thread.join(timeout=0.001)
1537 in_flight['threads'] -= 1
1538 finished.append(sum(thread.result))
1539 callback(thread.i, request_count, end=True)
1540
1541 q = Queue(max_threads)
1542 prod_thread = threading.Thread(target=producer,
1543 args=(q, requests, request_count))
1544 cons_thread = threading.Thread(target=consumer,
1545 args=(q, request_count))
1546 start = timeit.default_timer()
1547 prod_thread.start()
1548 cons_thread.start()
1549 _is_alive = thread_is_alive
1550 while _is_alive(prod_thread):
1551 prod_thread.join(timeout=0.001)
1552 while _is_alive(cons_thread):
1553 cons_thread.join(timeout=0.001)
1554
1555 stop = timeit.default_timer()
1556 self.results.bytes_received = sum(finished)
1557 self.results.download = (
1558 (self.results.bytes_received / (stop - start)) * 8.0
1559 )
1560 if self.results.download > 100000:
1561 self.config['threads']['upload'] = 8
1562 return self.results.download
1563
1564 def upload(self, callback=do_nothing, pre_allocate=True, threads=None):
1565 """Test upload speed against speedtest.net
1566 A ``threads`` value of ``None`` will fall back to those dictated
1567 by the speedtest.net configuration
1568 """
1569
1570 sizes = []
1571
1572 for size in self.config['sizes']['upload']:
1573 for _ in range(0, self.config['counts']['upload']):
1574 sizes.append(size)
1575
1576 # request_count = len(sizes)
1577 request_count = self.config['upload_max']
1578
1579 requests = []
1580 for i, size in enumerate(sizes):
1581 # We set ``0`` for ``start`` and handle setting the actual
1582 # ``start`` in ``HTTPUploader`` to get better measurements
1583 data = HTTPUploaderData(
1584 size,
1585 0,
1586 self.config['length']['upload'],
1587 shutdown_event=self._shutdown_event
1588 )
1589 if pre_allocate:
1590 data.pre_allocate()
1591
1592 headers = {'Content-length': size}
1593 requests.append(
1594 (
1595 build_request(self.best['url'], data, secure=self._secure,
1596 headers=headers),
1597 size
1598 )
1599 )
1600
1601 max_threads = threads or self.config['threads']['upload']
1602 in_flight = {'threads': 0}
1603
1604 def producer(q, requests, request_count):
1605 for i, request in enumerate(requests[:request_count]):
1606 thread = HTTPUploader(
1607 i,
1608 request[0],
1609 start,
1610 request[1],
1611 self.config['length']['upload'],
1612 opener=self._opener,
1613 shutdown_event=self._shutdown_event
1614 )
1615 while in_flight['threads'] >= max_threads:
1616 timeit.time.sleep(0.001)
1617 thread.start()
1618 q.put(thread, True)
1619 in_flight['threads'] += 1
1620 callback(i, request_count, start=True)
1621
1622 finished = []
1623
1624 def consumer(q, request_count):
1625 _is_alive = thread_is_alive
1626 while len(finished) < request_count:
1627 thread = q.get(True)
1628 while _is_alive(thread):
1629 thread.join(timeout=0.001)
1630 in_flight['threads'] -= 1
1631 finished.append(thread.result)
1632 callback(thread.i, request_count, end=True)
1633
1634 q = Queue(threads or self.config['threads']['upload'])
1635 prod_thread = threading.Thread(target=producer,
1636 args=(q, requests, request_count))
1637 cons_thread = threading.Thread(target=consumer,
1638 args=(q, request_count))
1639 start = timeit.default_timer()
1640 prod_thread.start()
1641 cons_thread.start()
1642 _is_alive = thread_is_alive
1643 while _is_alive(prod_thread):
1644 prod_thread.join(timeout=0.1)
1645 while _is_alive(cons_thread):
1646 cons_thread.join(timeout=0.1)
1647
1648 stop = timeit.default_timer()
1649 self.results.bytes_sent = sum(finished)
1650 self.results.upload = (
1651 (self.results.bytes_sent / (stop - start)) * 8.0
1652 )
1653 return self.results.upload
1654
1655
1656def ctrl_c(shutdown_event):
1657 """Catch Ctrl-C key sequence and set a SHUTDOWN_EVENT for our threaded
1658 operations
1659 """
1660 def inner(signum, frame):
1661 shutdown_event.set()
1662 printer('\nCancelling...', error=True)
1663 sys.exit(0)
1664 return inner
1665
1666
1667def version():
1668 """Print the version"""
1669
1670 printer('speedtest-cli %s' % __version__)
1671 printer('Python %s' % sys.version.replace('\n', ''))
1672 sys.exit(0)
1673
1674
1675def csv_header(delimiter=','):
1676 """Print the CSV Headers"""
1677
1678 printer(SpeedtestResults.csv_header(delimiter=delimiter))
1679 sys.exit(0)
1680
1681
1682def parse_args():
1683 """Function to handle building and parsing of command line arguments"""
1684 description = (
1685 'Command line interface for testing internet bandwidth using '
1686 'speedtest.net.\n'
1687 '------------------------------------------------------------'
1688 '--------------\n'
1689 'https://github.com/sivel/speedtest-cli')
1690
1691 parser = ArgParser(description=description)
1692 # Give optparse.OptionParser an `add_argument` method for
1693 # compatibility with argparse.ArgumentParser
1694 try:
1695 parser.add_argument = parser.add_option
1696 except AttributeError:
1697 pass
1698 parser.add_argument('--no-download', dest='download', default=True,
1699 action='store_const', const=False,
1700 help='Do not perform download test')
1701 parser.add_argument('--no-upload', dest='upload', default=True,
1702 action='store_const', const=False,
1703 help='Do not perform upload test')
1704 parser.add_argument('--single', default=False, action='store_true',
1705 help='Only use a single connection instead of '
1706 'multiple. This simulates a typical file '
1707 'transfer.')
1708 parser.add_argument('--bytes', dest='units', action='store_const',
1709 const=('byte', 8), default=('bit', 1),
1710 help='Display values in bytes instead of bits. Does '
1711 'not affect the image generated by --share, nor '
1712 'output from --json or --csv')
1713 parser.add_argument('--share', action='store_true',
1714 help='Generate and provide a URL to the speedtest.net '
1715 'share results image, not displayed with --csv')
1716 parser.add_argument('--simple', action='store_true', default=False,
1717 help='Suppress verbose output, only show basic '
1718 'information')
1719 parser.add_argument('--csv', action='store_true', default=False,
1720 help='Suppress verbose output, only show basic '
1721 'information in CSV format. Speeds listed in '
1722 'bit/s and not affected by --bytes')
1723 parser.add_argument('--csv-delimiter', default=',', type=PARSER_TYPE_STR,
1724 help='Single character delimiter to use in CSV '
1725 'output. Default ","')
1726 parser.add_argument('--csv-header', action='store_true', default=False,
1727 help='Print CSV headers')
1728 parser.add_argument('--json', action='store_true', default=False,
1729 help='Suppress verbose output, only show basic '
1730 'information in JSON format. Speeds listed in '
1731 'bit/s and not affected by --bytes')
1732 parser.add_argument('--list', action='store_true',
1733 help='Display a list of speedtest.net servers '
1734 'sorted by distance')
1735 parser.add_argument('--server', type=PARSER_TYPE_INT, action='append',
1736 help='Specify a server ID to test against. Can be '
1737 'supplied multiple times')
1738 parser.add_argument('--exclude', type=PARSER_TYPE_INT, action='append',
1739 help='Exclude a server from selection. Can be '
1740 'supplied multiple times')
1741 parser.add_argument('--mini', help='URL of the Speedtest Mini server')
1742 parser.add_argument('--source', help='Source IP address to bind to')
1743 parser.add_argument('--timeout', default=10, type=PARSER_TYPE_FLOAT,
1744 help='HTTP timeout in seconds. Default 10')
1745 parser.add_argument('--secure', action='store_true',
1746 help='Use HTTPS instead of HTTP when communicating '
1747 'with speedtest.net operated servers')
1748 parser.add_argument('--no-pre-allocate', dest='pre_allocate',
1749 action='store_const', default=True, const=False,
1750 help='Do not pre allocate upload data. Pre allocation '
1751 'is enabled by default to improve upload '
1752 'performance. To support systems with '
1753 'insufficient memory, use this option to avoid a '
1754 'MemoryError')
1755 parser.add_argument('--version', action='store_true',
1756 help='Show the version number and exit')
1757 parser.add_argument('--debug', action='store_true',
1758 help=ARG_SUPPRESS, default=ARG_SUPPRESS)
1759
1760 options = parser.parse_args()
1761 if isinstance(options, tuple):
1762 args = options[0]
1763 else:
1764 args = options
1765 return args
1766
1767
1768def validate_optional_args(args):
1769 """Check if an argument was provided that depends on a module that may
1770 not be part of the Python standard library.
1771 If such an argument is supplied, and the module does not exist, exit
1772 with an error stating which module is missing.
1773 """
1774 optional_args = {
1775 'json': ('json/simplejson python module', json),
1776 'secure': ('SSL support', HTTPSConnection),
1777 }
1778
1779 for arg, info in optional_args.items():
1780 if getattr(args, arg, False) and info[1] is None:
1781 raise SystemExit('%s is not installed. --%s is '
1782 'unavailable' % (info[0], arg))
1783
1784
1785def printer(string, quiet=False, debug=False, error=False, **kwargs):
1786 """Helper function print a string with various features"""
1787
1788 if debug and not DEBUG:
1789 return
1790
1791 if debug:
1792 if sys.stdout.isatty():
1793 out = '\033[1;30mDEBUG: %s\033[0m' % string
1794 else:
1795 out = 'DEBUG: %s' % string
1796 else:
1797 out = string
1798
1799 if error:
1800 kwargs['file'] = sys.stderr
1801
1802 if not quiet:
1803 print_(out, **kwargs)
1804
1805
1806def shell():
1807 """Run the full speedtest.net test"""
1808
1809 global DEBUG
1810 shutdown_event = threading.Event()
1811
1812 signal.signal(signal.SIGINT, ctrl_c(shutdown_event))
1813
1814 args = parse_args()
1815
1816 # Print the version and exit
1817 if args.version:
1818 version()
1819
1820 if not args.download and not args.upload:
1821 raise SpeedtestCLIError('Cannot supply both --no-download and '
1822 '--no-upload')
1823
1824 if len(args.csv_delimiter) != 1:
1825 raise SpeedtestCLIError('--csv-delimiter must be a single character')
1826
1827 if args.csv_header:
1828 csv_header(args.csv_delimiter)
1829
1830 validate_optional_args(args)
1831
1832 debug = getattr(args, 'debug', False)
1833 if debug == 'SUPPRESSHELP':
1834 debug = False
1835 if debug:
1836 DEBUG = True
1837
1838 if args.simple or args.csv or args.json:
1839 quiet = True
1840 else:
1841 quiet = False
1842
1843 if args.csv or args.json:
1844 machine_format = True
1845 else:
1846 machine_format = False
1847
1848 # Don't set a callback if we are running quietly
1849 if quiet or debug:
1850 callback = do_nothing
1851 else:
1852 callback = print_dots(shutdown_event)
1853
1854 printer('Retrieving speedtest.net configuration...', quiet)
1855 try:
1856 speedtest = Speedtest(
1857 source_address=args.source,
1858 timeout=args.timeout,
1859 secure=args.secure
1860 )
1861 except (ConfigRetrievalError,) + HTTP_ERRORS:
1862 printer('Cannot retrieve speedtest configuration', error=True)
1863 raise SpeedtestCLIError(get_exception())
1864
1865 if args.list:
1866 try:
1867 speedtest.get_servers()
1868 except (ServersRetrievalError,) + HTTP_ERRORS:
1869 printer('Cannot retrieve speedtest server list', error=True)
1870 raise SpeedtestCLIError(get_exception())
1871
1872 for _, servers in sorted(speedtest.servers.items()):
1873 for server in servers:
1874 line = ('%(id)5s) %(sponsor)s (%(name)s, %(country)s) '
1875 '[%(d)0.2f km]' % server)
1876 try:
1877 printer(line)
1878 except IOError:
1879 e = get_exception()
1880 if e.errno != errno.EPIPE:
1881 raise
1882 sys.exit(0)
1883
1884 printer('Testing from %(isp)s (%(ip)s)...' % speedtest.config['client'],
1885 quiet)
1886
1887 if not args.mini:
1888 printer('Retrieving speedtest.net server list...', quiet)
1889 try:
1890 speedtest.get_servers(servers=args.server, exclude=args.exclude)
1891 except NoMatchedServers:
1892 raise SpeedtestCLIError(
1893 'No matched servers: %s' %
1894 ', '.join('%s' % s for s in args.server)
1895 )
1896 except (ServersRetrievalError,) + HTTP_ERRORS:
1897 printer('Cannot retrieve speedtest server list', error=True)
1898 raise SpeedtestCLIError(get_exception())
1899 except InvalidServerIDType:
1900 raise SpeedtestCLIError(
1901 '%s is an invalid server type, must '
1902 'be an int' % ', '.join('%s' % s for s in args.server)
1903 )
1904
1905 if args.server and len(args.server) == 1:
1906 printer('Retrieving information for the selected server...', quiet)
1907 else:
1908 printer('Selecting best server based on ping...', quiet)
1909 speedtest.get_best_server()
1910 elif args.mini:
1911 speedtest.get_best_server(speedtest.set_mini_server(args.mini))
1912
1913 results = speedtest.results
1914
1915 printer('Hosted by %(sponsor)s (%(name)s) [%(d)0.2f km]: '
1916 '%(latency)s ms' % results.server, quiet)
1917
1918 if args.download:
1919 printer('Testing download speed', quiet,
1920 end=('', '\n')[bool(debug)])
1921 speedtest.download(
1922 callback=callback,
1923 threads=(None, 1)[args.single]
1924 )
1925 printer('Download: %0.2f M%s/s' %
1926 ((results.download / 1000.0 / 1000.0) / args.units[1],
1927 args.units[0]),
1928 quiet)
1929 else:
1930 printer('Skipping download test', quiet)
1931
1932 if args.upload:
1933 printer('Testing upload speed', quiet,
1934 end=('', '\n')[bool(debug)])
1935 speedtest.upload(
1936 callback=callback,
1937 pre_allocate=args.pre_allocate,
1938 threads=(None, 1)[args.single]
1939 )
1940 printer('Upload: %0.2f M%s/s' %
1941 ((results.upload / 1000.0 / 1000.0) / args.units[1],
1942 args.units[0]),
1943 quiet)
1944 else:
1945 printer('Skipping upload test', quiet)
1946
1947 printer('Results:\n%r' % results.dict(), debug=True)
1948
1949 if not args.simple and args.share:
1950 results.share()
1951
1952 if args.simple:
1953 printer('Ping: %s ms\nDownload: %0.2f M%s/s\nUpload: %0.2f M%s/s' %
1954 (results.ping,
1955 (results.download / 1000.0 / 1000.0) / args.units[1],
1956 args.units[0],
1957 (results.upload / 1000.0 / 1000.0) / args.units[1],
1958 args.units[0]))
1959 elif args.csv:
1960 printer(results.csv(delimiter=args.csv_delimiter))
1961 elif args.json:
1962 printer(results.json())
1963
1964 if args.share and not machine_format:
1965 printer('Share results: %s' % results.share())
1966
1967
1968def main():
1969 try:
1970 shell()
1971 except KeyboardInterrupt:
1972 printer('\nCancelling...', error=True)
1973 except (SpeedtestException, SystemExit):
1974 e = get_exception()
1975 # Ignore a successful exit, or argparse exit
1976 if getattr(e, 'code', 1) not in (0, 2):
1977 msg = '%s' % e
1978 if not msg:
1979 msg = '%r' % e
1980 raise SystemExit('ERROR: %s' % msg)
1981
1982
1983if __name__ == '__main__':
1984 main()