· 6 years ago · Sep 23, 2019, 09:40 AM
1import os
2import re
3import csv
4import sys
5import math
6import errno
7import signal
8import socket
9import timeit
10import datetime
11import platform
12import threading
13import xml.parsers.expat
14
15try:
16 import gzip
17 GZIP_BASE = gzip.GzipFile
18except ImportError:
19 gzip = None
20 GZIP_BASE = object
21
22__version__ = '2.1.2'
23
24
25class FakeShutdownEvent(object):
26 """Class to fake a threading.Event.isSet so that users of this module
27 are not required to register their own threading.Event()
28 """
29
30 @staticmethod
31 def isSet():
32 "Dummy method to always return false"""
33 return False
34
35
36# Some global variables we use
37DEBUG = False
38_GLOBAL_DEFAULT_TIMEOUT = object()
39PY25PLUS = sys.version_info[:2] >= (2, 5)
40PY26PLUS = sys.version_info[:2] >= (2, 6)
41PY32PLUS = sys.version_info[:2] >= (3, 2)
42
43# Begin import game to handle Python 2 and Python 3
44try:
45 import json
46except ImportError:
47 try:
48 import simplejson as json
49 except ImportError:
50 json = None
51
52try:
53 import xml.etree.ElementTree as ET
54 try:
55 from xml.etree.ElementTree import _Element as ET_Element
56 except ImportError:
57 pass
58except ImportError:
59 from xml.dom import minidom as DOM
60 from xml.parsers.expat import ExpatError
61 ET = None
62
63try:
64 from urllib2 import (urlopen, Request, HTTPError, URLError,
65 AbstractHTTPHandler, ProxyHandler,
66 HTTPDefaultErrorHandler, HTTPRedirectHandler,
67 HTTPErrorProcessor, OpenerDirector)
68except ImportError:
69 from urllib.request import (urlopen, Request, HTTPError, URLError,
70 AbstractHTTPHandler, ProxyHandler,
71 HTTPDefaultErrorHandler, HTTPRedirectHandler,
72 HTTPErrorProcessor, OpenerDirector)
73
74try:
75 from httplib import HTTPConnection, BadStatusLine
76except ImportError:
77 from http.client import HTTPConnection, BadStatusLine
78
79try:
80 from httplib import HTTPSConnection
81except ImportError:
82 try:
83 from http.client import HTTPSConnection
84 except ImportError:
85 HTTPSConnection = None
86
87try:
88 from httplib import FakeSocket
89except ImportError:
90 FakeSocket = None
91
92try:
93 from Queue import Queue
94except ImportError:
95 from queue import Queue
96
97try:
98 from urlparse import urlparse
99except ImportError:
100 from urllib.parse import urlparse
101
102try:
103 from urlparse import parse_qs
104except ImportError:
105 try:
106 from urllib.parse import parse_qs
107 except ImportError:
108 from cgi import parse_qs
109
110try:
111 from hashlib import md5
112except ImportError:
113 from md5 import md5
114
115try:
116 from argparse import ArgumentParser as ArgParser
117 from argparse import SUPPRESS as ARG_SUPPRESS
118 PARSER_TYPE_INT = int
119 PARSER_TYPE_STR = str
120 PARSER_TYPE_FLOAT = float
121except ImportError:
122 from optparse import OptionParser as ArgParser
123 from optparse import SUPPRESS_HELP as ARG_SUPPRESS
124 PARSER_TYPE_INT = 'int'
125 PARSER_TYPE_STR = 'string'
126 PARSER_TYPE_FLOAT = 'float'
127
128try:
129 from cStringIO import StringIO
130 BytesIO = None
131except ImportError:
132 try:
133 from StringIO import StringIO
134 BytesIO = None
135 except ImportError:
136 from io import StringIO, BytesIO
137
138try:
139 import __builtin__
140except ImportError:
141 import builtins
142 from io import TextIOWrapper, FileIO
143
144 class _Py3Utf8Output(TextIOWrapper):
145 """UTF-8 encoded wrapper around stdout for py3, to override
146 ASCII stdout
147 """
148 def __init__(self, f, **kwargs):
149 buf = FileIO(f.fileno(), 'w')
150 super(_Py3Utf8Output, self).__init__(
151 buf,
152 encoding='utf8',
153 errors='strict'
154 )
155
156 def write(self, s):
157 super(_Py3Utf8Output, self).write(s)
158 self.flush()
159
160 _py3_print = getattr(builtins, 'print')
161 try:
162 _py3_utf8_stdout = _Py3Utf8Output(sys.stdout)
163 _py3_utf8_stderr = _Py3Utf8Output(sys.stderr)
164 except OSError:
165 # sys.stdout/sys.stderr is not a compatible stdout/stderr object
166 # just use it and hope things go ok
167 _py3_utf8_stdout = sys.stdout
168 _py3_utf8_stderr = sys.stderr
169
170 def to_utf8(v):
171 """No-op encode to utf-8 for py3"""
172 return v
173
174 def print_(*args, **kwargs):
175 """Wrapper function for py3 to print, with a utf-8 encoded stdout"""
176 if kwargs.get('file') == sys.stderr:
177 kwargs['file'] = _py3_utf8_stderr
178 else:
179 kwargs['file'] = kwargs.get('file', _py3_utf8_stdout)
180 _py3_print(*args, **kwargs)
181else:
182 del __builtin__
183
184 def to_utf8(v):
185 """Encode value to utf-8 if possible for py2"""
186 try:
187 return v.encode('utf8', 'strict')
188 except AttributeError:
189 return v
190
191 def print_(*args, **kwargs):
192 """The new-style print function for Python 2.4 and 2.5.
193
194 Taken from https://pypi.python.org/pypi/six/
195
196 Modified to set encoding to UTF-8 always, and to flush after write
197 """
198 fp = kwargs.pop("file", sys.stdout)
199 if fp is None:
200 return
201
202 def write(data):
203 if not isinstance(data, basestring):
204 data = str(data)
205 # If the file has an encoding, encode unicode with it.
206 encoding = 'utf8' # Always trust UTF-8 for output
207 if (isinstance(fp, file) and
208 isinstance(data, unicode) and
209 encoding is not None):
210 errors = getattr(fp, "errors", None)
211 if errors is None:
212 errors = "strict"
213 data = data.encode(encoding, errors)
214 fp.write(data)
215 fp.flush()
216 want_unicode = False
217 sep = kwargs.pop("sep", None)
218 if sep is not None:
219 if isinstance(sep, unicode):
220 want_unicode = True
221 elif not isinstance(sep, str):
222 raise TypeError("sep must be None or a string")
223 end = kwargs.pop("end", None)
224 if end is not None:
225 if isinstance(end, unicode):
226 want_unicode = True
227 elif not isinstance(end, str):
228 raise TypeError("end must be None or a string")
229 if kwargs:
230 raise TypeError("invalid keyword arguments to print()")
231 if not want_unicode:
232 for arg in args:
233 if isinstance(arg, unicode):
234 want_unicode = True
235 break
236 if want_unicode:
237 newline = unicode("\n")
238 space = unicode(" ")
239 else:
240 newline = "\n"
241 space = " "
242 if sep is None:
243 sep = space
244 if end is None:
245 end = newline
246 for i, arg in enumerate(args):
247 if i:
248 write(sep)
249 write(arg)
250 write(end)
251
252if PY32PLUS:
253 etree_iter = ET.Element.iter
254elif PY25PLUS:
255 etree_iter = ET_Element.getiterator
256
257if PY26PLUS:
258 thread_is_alive = threading.Thread.is_alive
259else:
260 thread_is_alive = threading.Thread.isAlive
261
262
263# Exception "constants" to support Python 2 through Python 3
264try:
265 import ssl
266 try:
267 CERT_ERROR = (ssl.CertificateError,)
268 except AttributeError:
269 CERT_ERROR = tuple()
270
271 HTTP_ERRORS = (
272 (HTTPError, URLError, socket.error, ssl.SSLError, BadStatusLine) +
273 CERT_ERROR
274 )
275except ImportError:
276 ssl = None
277 HTTP_ERRORS = (HTTPError, URLError, socket.error, BadStatusLine)
278
279
280class SpeedtestException(Exception):
281 """Base exception for this module"""
282
283
284class SpeedtestCLIError(SpeedtestException):
285 """Generic exception for raising errors during CLI operation"""
286
287
288class SpeedtestHTTPError(SpeedtestException):
289 """Base HTTP exception for this module"""
290
291
292class SpeedtestConfigError(SpeedtestException):
293 """Configuration XML is invalid"""
294
295
296class SpeedtestServersError(SpeedtestException):
297 """Servers XML is invalid"""
298
299
300class ConfigRetrievalError(SpeedtestHTTPError):
301 """Could not retrieve config.php"""
302
303
304class ServersRetrievalError(SpeedtestHTTPError):
305 """Could not retrieve speedtest-servers.php"""
306
307
308class InvalidServerIDType(SpeedtestException):
309 """Server ID used for filtering was not an integer"""
310
311
312class NoMatchedServers(SpeedtestException):
313 """No servers matched when filtering"""
314
315
316class SpeedtestMiniConnectFailure(SpeedtestException):
317 """Could not connect to the provided speedtest mini server"""
318
319
320class InvalidSpeedtestMiniServer(SpeedtestException):
321 """Server provided as a speedtest mini server does not actually appear
322 to be a speedtest mini server
323 """
324
325
326class ShareResultsConnectFailure(SpeedtestException):
327 """Could not connect to speedtest.net API to POST results"""
328
329
330class ShareResultsSubmitFailure(SpeedtestException):
331 """Unable to successfully POST results to speedtest.net API after
332 connection
333 """
334
335
336class SpeedtestUploadTimeout(SpeedtestException):
337 """testlength configuration reached during upload
338 Used to ensure the upload halts when no additional data should be sent
339 """
340
341
342class SpeedtestBestServerFailure(SpeedtestException):
343 """Unable to determine best server"""
344
345
346class SpeedtestMissingBestServer(SpeedtestException):
347 """get_best_server not called or not able to determine best server"""
348
349
350def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
351 source_address=None):
352 """Connect to *address* and return the socket object.
353
354 Convenience function. Connect to *address* (a 2-tuple ``(host,
355 port)``) and return the socket object. Passing the optional
356 *timeout* parameter will set the timeout on the socket instance
357 before attempting to connect. If no *timeout* is supplied, the
358 global default timeout setting returned by :func:`getdefaulttimeout`
359 is used. If *source_address* is set it must be a tuple of (host, port)
360 for the socket to bind as a source address before making the connection.
361 An host of '' or port 0 tells the OS to use the default.
362
363 Largely vendored from Python 2.7, modified to work with Python 2.4
364 """
365
366 host, port = address
367 err = None
368 for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
369 af, socktype, proto, canonname, sa = res
370 sock = None
371 try:
372 sock = socket.socket(af, socktype, proto)
373 if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
374 sock.settimeout(float(timeout))
375 if source_address:
376 sock.bind(source_address)
377 sock.connect(sa)
378 return sock
379
380 except socket.error:
381 err = get_exception()
382 if sock is not None:
383 sock.close()
384
385 if err is not None:
386 raise err
387 else:
388 raise socket.error("getaddrinfo returns an empty list")
389
390
391class SpeedtestHTTPConnection(HTTPConnection):
392 """Custom HTTPConnection to support source_address across
393 Python 2.4 - Python 3
394 """
395 def __init__(self, *args, **kwargs):
396 source_address = kwargs.pop('source_address', None)
397 timeout = kwargs.pop('timeout', 10)
398
399 self._tunnel_host = None
400
401 HTTPConnection.__init__(self, *args, **kwargs)
402
403 self.source_address = source_address
404 self.timeout = timeout
405
406 def connect(self):
407 """Connect to the host and port specified in __init__."""
408 try:
409 self.sock = socket.create_connection(
410 (self.host, self.port),
411 self.timeout,
412 self.source_address
413 )
414 except (AttributeError, TypeError):
415 self.sock = create_connection(
416 (self.host, self.port),
417 self.timeout,
418 self.source_address
419 )
420
421 if self._tunnel_host:
422 self._tunnel()
423
424
425if HTTPSConnection:
426 class SpeedtestHTTPSConnection(HTTPSConnection):
427 """Custom HTTPSConnection to support source_address across
428 Python 2.4 - Python 3
429 """
430 default_port = 443
431
432 def __init__(self, *args, **kwargs):
433 source_address = kwargs.pop('source_address', None)
434 timeout = kwargs.pop('timeout', 10)
435
436 self._tunnel_host = None
437
438 HTTPSConnection.__init__(self, *args, **kwargs)
439
440 self.timeout = timeout
441 self.source_address = source_address
442
443 def connect(self):
444 "Connect to a host on a given (SSL) port."
445 try:
446 self.sock = socket.create_connection(
447 (self.host, self.port),
448 self.timeout,
449 self.source_address
450 )
451 except (AttributeError, TypeError):
452 self.sock = create_connection(
453 (self.host, self.port),
454 self.timeout,
455 self.source_address
456 )
457
458 if self._tunnel_host:
459 self._tunnel()
460
461 if ssl:
462 try:
463 kwargs = {}
464 if hasattr(ssl, 'SSLContext'):
465 if self._tunnel_host:
466 kwargs['server_hostname'] = self._tunnel_host
467 else:
468 kwargs['server_hostname'] = self.host
469 self.sock = self._context.wrap_socket(self.sock, **kwargs)
470 except AttributeError:
471 self.sock = ssl.wrap_socket(self.sock)
472 try:
473 self.sock.server_hostname = self.host
474 except AttributeError:
475 pass
476 elif FakeSocket:
477 # Python 2.4/2.5 support
478 try:
479 self.sock = FakeSocket(self.sock, socket.ssl(self.sock))
480 except AttributeError:
481 raise SpeedtestException(
482 'This version of Python does not support HTTPS/SSL '
483 'functionality'
484 )
485 else:
486 raise SpeedtestException(
487 'This version of Python does not support HTTPS/SSL '
488 'functionality'
489 )
490
491
492def _build_connection(connection, source_address, timeout, context=None):
493 """Cross Python 2.4 - Python 3 callable to build an ``HTTPConnection`` or
494 ``HTTPSConnection`` with the args we need
495
496 Called from ``http(s)_open`` methods of ``SpeedtestHTTPHandler`` or
497 ``SpeedtestHTTPSHandler``
498 """
499 def inner(host, **kwargs):
500 kwargs.update({
501 'source_address': source_address,
502 'timeout': timeout
503 })
504 if context:
505 kwargs['context'] = context
506 return connection(host, **kwargs)
507 return inner
508
509
510class SpeedtestHTTPHandler(AbstractHTTPHandler):
511 """Custom ``HTTPHandler`` that can build a ``HTTPConnection`` with the
512 args we need for ``source_address`` and ``timeout``
513 """
514 def __init__(self, debuglevel=0, source_address=None, timeout=10):
515 AbstractHTTPHandler.__init__(self, debuglevel)
516 self.source_address = source_address
517 self.timeout = timeout
518
519 def http_open(self, req):
520 return self.do_open(
521 _build_connection(
522 SpeedtestHTTPConnection,
523 self.source_address,
524 self.timeout
525 ),
526 req
527 )
528
529 http_request = AbstractHTTPHandler.do_request_
530
531
532class SpeedtestHTTPSHandler(AbstractHTTPHandler):
533 """Custom ``HTTPSHandler`` that can build a ``HTTPSConnection`` with the
534 args we need for ``source_address`` and ``timeout``
535 """
536 def __init__(self, debuglevel=0, context=None, source_address=None,
537 timeout=10):
538 AbstractHTTPHandler.__init__(self, debuglevel)
539 self._context = context
540 self.source_address = source_address
541 self.timeout = timeout
542
543 def https_open(self, req):
544 return self.do_open(
545 _build_connection(
546 SpeedtestHTTPSConnection,
547 self.source_address,
548 self.timeout,
549 context=self._context,
550 ),
551 req
552 )
553
554 https_request = AbstractHTTPHandler.do_request_
555
556
557def build_opener(source_address=None, timeout=10):
558 """Function similar to ``urllib2.build_opener`` that will build
559 an ``OpenerDirector`` with the explicit handlers we want,
560 ``source_address`` for binding, ``timeout`` and our custom
561 `User-Agent`
562 """
563
564 printer('Timeout set to %d' % timeout, debug=True)
565
566 if source_address:
567 source_address_tuple = (source_address, 0)
568 printer('Binding to source address: %r' % (source_address_tuple,),
569 debug=True)
570 else:
571 source_address_tuple = None
572
573 handlers = [
574 ProxyHandler(),
575 SpeedtestHTTPHandler(source_address=source_address_tuple,
576 timeout=timeout),
577 SpeedtestHTTPSHandler(source_address=source_address_tuple,
578 timeout=timeout),
579 HTTPDefaultErrorHandler(),
580 HTTPRedirectHandler(),
581 HTTPErrorProcessor()
582 ]
583
584 opener = OpenerDirector()
585 opener.addheaders = [('User-agent', build_user_agent())]
586
587 for handler in handlers:
588 opener.add_handler(handler)
589
590 return opener
591
592
593class GzipDecodedResponse(GZIP_BASE):
594 """A file-like object to decode a response encoded with the gzip
595 method, as described in RFC 1952.
596
597 Largely copied from ``xmlrpclib``/``xmlrpc.client`` and modified
598 to work for py2.4-py3
599 """
600 def __init__(self, response):
601 # response doesn't support tell() and read(), required by
602 # GzipFile
603 if not gzip:
604 raise SpeedtestHTTPError('HTTP response body is gzip encoded, '
605 'but gzip support is not available')
606 IO = BytesIO or StringIO
607 self.io = IO()
608 while 1:
609 chunk = response.read(1024)
610 if len(chunk) == 0:
611 break
612 self.io.write(chunk)
613 self.io.seek(0)
614 gzip.GzipFile.__init__(self, mode='rb', fileobj=self.io)
615
616 def close(self):
617 try:
618 gzip.GzipFile.close(self)
619 finally:
620 self.io.close()
621
622
623def get_exception():
624 """Helper function to work with py2.4-py3 for getting the current
625 exception in a try/except block
626 """
627 return sys.exc_info()[1]
628
629
630def distance(origin, destination):
631 """Determine distance between 2 sets of [lat,lon] in km"""
632
633 lat1, lon1 = origin
634 lat2, lon2 = destination
635 radius = 6371 # km
636
637 dlat = math.radians(lat2 - lat1)
638 dlon = math.radians(lon2 - lon1)
639 a = (math.sin(dlat / 2) * math.sin(dlat / 2) +
640 math.cos(math.radians(lat1)) *
641 math.cos(math.radians(lat2)) * math.sin(dlon / 2) *
642 math.sin(dlon / 2))
643 c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
644 d = radius * c
645
646 return d
647
648
649def build_user_agent():
650 """Build a Mozilla/5.0 compatible User-Agent string"""
651
652 ua_tuple = (
653 'Mozilla/5.0',
654 '(%s; U; %s; en-us)' % (platform.platform(),
655 platform.architecture()[0]),
656 'Python/%s' % platform.python_version(),
657 '(KHTML, like Gecko)',
658 'speedtest-cli/%s' % __version__
659 )
660 user_agent = ' '.join(ua_tuple)
661 printer('User-Agent: %s' % user_agent, debug=True)
662 return user_agent
663
664
665def build_request(url, data=None, headers=None, bump='0', secure=False):
666 """Build a urllib2 request object
667
668 This function automatically adds a User-Agent header to all requests
669
670 """
671
672 if not headers:
673 headers = {}
674
675 if url[0] == ':':
676 scheme = ('http', 'https')[bool(secure)]
677 schemed_url = '%s%s' % (scheme, url)
678 else:
679 schemed_url = url
680
681 if '?' in url:
682 delim = '&'
683 else:
684 delim = '?'
685
686 # WHO YOU GONNA CALL? CACHE BUSTERS!
687 final_url = '%s%sx=%s.%s' % (schemed_url, delim,
688 int(timeit.time.time() * 1000),
689 bump)
690
691 headers.update({
692 'Cache-Control': 'no-cache',
693 })
694
695 printer('%s %s' % (('GET', 'POST')[bool(data)], final_url),
696 debug=True)
697
698 return Request(final_url, data=data, headers=headers)
699
700
701def catch_request(request, opener=None):
702 """Helper function to catch common exceptions encountered when
703 establishing a connection with a HTTP/HTTPS request
704
705 """
706
707 if opener:
708 _open = opener.open
709 else:
710 _open = urlopen
711
712 try:
713 uh = _open(request)
714 if request.get_full_url() != uh.geturl():
715 printer('Redirected to %s' % uh.geturl(), debug=True)
716 return uh, False
717 except HTTP_ERRORS:
718 e = get_exception()
719 return None, e
720
721
722def get_response_stream(response):
723 """Helper function to return either a Gzip reader if
724 ``Content-Encoding`` is ``gzip`` otherwise the response itself
725
726 """
727
728 try:
729 getheader = response.headers.getheader
730 except AttributeError:
731 getheader = response.getheader
732
733 if getheader('content-encoding') == 'gzip':
734 return GzipDecodedResponse(response)
735
736 return response
737
738
739def get_attributes_by_tag_name(dom, tag_name):
740 """Retrieve an attribute from an XML document and return it in a
741 consistent format
742
743 Only used with xml.dom.minidom, which is likely only to be used
744 with python versions older than 2.5
745 """
746 elem = dom.getElementsByTagName(tag_name)[0]
747 return dict(list(elem.attributes.items()))
748
749
750def print_dots(shutdown_event):
751 """Built in callback function used by Thread classes for printing
752 status
753 """
754 def inner(current, total, start=False, end=False):
755 if shutdown_event.isSet():
756 return
757
758 sys.stdout.write('.')
759 if current + 1 == total and end is True:
760 sys.stdout.write('\n')
761 sys.stdout.flush()
762 return inner
763
764
765def do_nothing(*args, **kwargs):
766 pass
767
768
769class HTTPDownloader(threading.Thread):
770 """Thread class for retrieving a URL"""
771
772 def __init__(self, i, request, start, timeout, opener=None,
773 shutdown_event=None):
774 threading.Thread.__init__(self)
775 self.request = request
776 self.result = [0]
777 self.starttime = start
778 self.timeout = timeout
779 self.i = i
780 if opener:
781 self._opener = opener.open
782 else:
783 self._opener = urlopen
784
785 if shutdown_event:
786 self._shutdown_event = shutdown_event
787 else:
788 self._shutdown_event = FakeShutdownEvent()
789
790 def run(self):
791 try:
792 if (timeit.default_timer() - self.starttime) <= self.timeout:
793 f = self._opener(self.request)
794 while (not self._shutdown_event.isSet() and
795 (timeit.default_timer() - self.starttime) <=
796 self.timeout):
797 self.result.append(len(f.read(10240)))
798 if self.result[-1] == 0:
799 break
800 f.close()
801 except IOError:
802 pass
803
804
805class HTTPUploaderData(object):
806 """File like object to improve cutting off the upload once the timeout
807 has been reached
808 """
809
810 def __init__(self, length, start, timeout, shutdown_event=None):
811 self.length = length
812 self.start = start
813 self.timeout = timeout
814
815 if shutdown_event:
816 self._shutdown_event = shutdown_event
817 else:
818 self._shutdown_event = FakeShutdownEvent()
819
820 self._data = None
821
822 self.total = [0]
823
824 def pre_allocate(self):
825 chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'
826 multiplier = int(round(int(self.length) / 36.0))
827 IO = BytesIO or StringIO
828 try:
829 self._data = IO(
830 ('content1=%s' %
831 (chars * multiplier)[0:int(self.length) - 9]
832 ).encode()
833 )
834 except MemoryError:
835 raise SpeedtestCLIError(
836 'Insufficient memory to pre-allocate upload data. Please '
837 'use --no-pre-allocate'
838 )
839
840 @property
841 def data(self):
842 if not self._data:
843 self.pre_allocate()
844 return self._data
845
846 def read(self, n=10240):
847 if ((timeit.default_timer() - self.start) <= self.timeout and
848 not self._shutdown_event.isSet()):
849 chunk = self.data.read(n)
850 self.total.append(len(chunk))
851 return chunk
852 else:
853 raise SpeedtestUploadTimeout()
854
855 def __len__(self):
856 return self.length
857
858
859class HTTPUploader(threading.Thread):
860 """Thread class for putting a URL"""
861
862 def __init__(self, i, request, start, size, timeout, opener=None,
863 shutdown_event=None):
864 threading.Thread.__init__(self)
865 self.request = request
866 self.request.data.start = self.starttime = start
867 self.size = size
868 self.result = None
869 self.timeout = timeout
870 self.i = i
871
872 if opener:
873 self._opener = opener.open
874 else:
875 self._opener = urlopen
876
877 if shutdown_event:
878 self._shutdown_event = shutdown_event
879 else:
880 self._shutdown_event = FakeShutdownEvent()
881
882 def run(self):
883 request = self.request
884 try:
885 if ((timeit.default_timer() - self.starttime) <= self.timeout and
886 not self._shutdown_event.isSet()):
887 try:
888 f = self._opener(request)
889 except TypeError:
890 # PY24 expects a string or buffer
891 # This also causes issues with Ctrl-C, but we will concede
892 # for the moment that Ctrl-C on PY24 isn't immediate
893 request = build_request(self.request.get_full_url(),
894 data=request.data.read(self.size))
895 f = self._opener(request)
896 f.read(11)
897 f.close()
898 self.result = sum(self.request.data.total)
899 else:
900 self.result = 0
901 except (IOError, SpeedtestUploadTimeout):
902 self.result = sum(self.request.data.total)
903
904
905class SpeedtestResults(object):
906 """Class for holding the results of a speedtest, including:
907
908 Download speed
909 Upload speed
910 Ping/Latency to test server
911 Data about server that the test was run against
912
913 Additionally this class can return a result data as a dictionary or CSV,
914 as well as submit a POST of the result data to the speedtest.net API
915 to get a share results image link.
916 """
917
918 def __init__(self, download=0, upload=0, ping=0, server=None, client=None,
919 opener=None, secure=False):
920 self.download = download
921 self.upload = upload
922 self.ping = ping
923 if server is None:
924 self.server = {}
925 else:
926 self.server = server
927 self.client = client or {}
928
929 self._share = None
930 self.timestamp = '%sZ' % datetime.datetime.utcnow().isoformat()
931 self.bytes_received = 0
932 self.bytes_sent = 0
933
934 if opener:
935 self._opener = opener
936 else:
937 self._opener = build_opener()
938
939 self._secure = secure
940
941 def __repr__(self):
942 return repr(self.dict())
943
944 def share(self):
945 """POST data to the speedtest.net API to obtain a share results
946 link
947 """
948
949 if self._share:
950 return self._share
951
952 download = int(round(self.download / 1000.0, 0))
953 ping = int(round(self.ping, 0))
954 upload = int(round(self.upload / 1000.0, 0))
955
956 # Build the request to send results back to speedtest.net
957 # We use a list instead of a dict because the API expects parameters
958 # in a certain order
959 api_data = [
960 'recommendedserverid=%s' % self.server['id'],
961 'ping=%s' % ping,
962 'screenresolution=',
963 'promo=',
964 'download=%s' % download,
965 'screendpi=',
966 'upload=%s' % upload,
967 'testmethod=http',
968 'hash=%s' % md5(('%s-%s-%s-%s' %
969 (ping, upload, download, '297aae72'))
970 .encode()).hexdigest(),
971 'touchscreen=none',
972 'startmode=pingselect',
973 'accuracy=1',
974 'bytesreceived=%s' % self.bytes_received,
975 'bytessent=%s' % self.bytes_sent,
976 'serverid=%s' % self.server['id'],
977 ]
978
979 headers = {'Referer': 'http://c.speedtest.net/flash/speedtest.swf'}
980 request = build_request('://www.speedtest.net/api/api.php',
981 data='&'.join(api_data).encode(),
982 headers=headers, secure=self._secure)
983 f, e = catch_request(request, opener=self._opener)
984 if e:
985 raise ShareResultsConnectFailure(e)
986
987 response = f.read()
988 code = f.code
989 f.close()
990
991 if int(code) != 200:
992 raise ShareResultsSubmitFailure('Could not submit results to '
993 'speedtest.net')
994
995 qsargs = parse_qs(response.decode())
996 resultid = qsargs.get('resultid')
997 if not resultid or len(resultid) != 1:
998 raise ShareResultsSubmitFailure('Could not submit results to '
999 'speedtest.net')
1000
1001 self._share = 'http://www.speedtest.net/result/%s.png' % resultid[0]
1002
1003 return self._share
1004
1005 def dict(self):
1006 """Return dictionary of result data"""
1007
1008 return {
1009 'download': self.download,
1010 'upload': self.upload,
1011 'ping': self.ping,
1012 'server': self.server,
1013 'timestamp': self.timestamp,
1014 'bytes_sent': self.bytes_sent,
1015 'bytes_received': self.bytes_received,
1016 'share': self._share,
1017 'client': self.client,
1018 }
1019
1020 @staticmethod
1021 def csv_header(delimiter=','):
1022 """Return CSV Headers"""
1023
1024 row = ['Server ID', 'Sponsor', 'Server Name', 'Timestamp', 'Distance',
1025 'Ping', 'Download', 'Upload', 'Share', 'IP Address']
1026 out = StringIO()
1027 writer = csv.writer(out, delimiter=delimiter, lineterminator='')
1028 writer.writerow([to_utf8(v) for v in row])
1029 return out.getvalue()
1030
1031 def csv(self, delimiter=','):
1032 """Return data in CSV format"""
1033
1034 data = self.dict()
1035 out = StringIO()
1036 writer = csv.writer(out, delimiter=delimiter, lineterminator='')
1037 row = [data['server']['id'], data['server']['sponsor'],
1038 data['server']['name'], data['timestamp'],
1039 data['server']['d'], data['ping'], data['download'],
1040 data['upload'], self._share or '', self.client['ip']]
1041 writer.writerow([to_utf8(v) for v in row])
1042 return out.getvalue()
1043
1044 def json(self, pretty=False):
1045 """Return data in JSON format"""
1046
1047 kwargs = {}
1048 if pretty:
1049 kwargs.update({
1050 'indent': 4,
1051 'sort_keys': True
1052 })
1053 return json.dumps(self.dict(), **kwargs)
1054
1055
1056class Speedtest(object):
1057 """Class for performing standard speedtest.net testing operations"""
1058
1059 def __init__(self, config=None, source_address=None, timeout=10,
1060 secure=False, shutdown_event=None):
1061 self.config = {}
1062
1063 self._source_address = source_address
1064 self._timeout = timeout
1065 self._opener = build_opener(source_address, timeout)
1066
1067 self._secure = secure
1068
1069 if shutdown_event:
1070 self._shutdown_event = shutdown_event
1071 else:
1072 self._shutdown_event = FakeShutdownEvent()
1073
1074 self.get_config()
1075 if config is not None:
1076 self.config.update(config)
1077
1078 self.servers = {}
1079 self.closest = []
1080 self._best = {}
1081
1082 self.results = SpeedtestResults(
1083 client=self.config['client'],
1084 opener=self._opener,
1085 secure=secure,
1086 )
1087
1088 @property
1089 def best(self):
1090 if not self._best:
1091 self.get_best_server()
1092 return self._best
1093
1094 def get_config(self):
1095 """Download the speedtest.net configuration and return only the data
1096 we are interested in
1097 """
1098
1099 headers = {}
1100 if gzip:
1101 headers['Accept-Encoding'] = 'gzip'
1102 request = build_request('://www.speedtest.net/speedtest-config.php',
1103 headers=headers, secure=self._secure)
1104 uh, e = catch_request(request, opener=self._opener)
1105 if e:
1106 raise ConfigRetrievalError(e)
1107 configxml_list = []
1108
1109 stream = get_response_stream(uh)
1110
1111 while 1:
1112 try:
1113 configxml_list.append(stream.read(1024))
1114 except (OSError, EOFError):
1115 raise ConfigRetrievalError(get_exception())
1116 if len(configxml_list[-1]) == 0:
1117 break
1118 stream.close()
1119 uh.close()
1120
1121 if int(uh.code) != 200:
1122 return None
1123
1124 configxml = ''.encode().join(configxml_list)
1125
1126 printer('Config XML:\n%s' % configxml, debug=True)
1127
1128 try:
1129 try:
1130 root = ET.fromstring(configxml)
1131 except ET.ParseError:
1132 e = get_exception()
1133 raise SpeedtestConfigError(
1134 'Malformed speedtest.net configuration: %s' % e
1135 )
1136 server_config = root.find('server-config').attrib
1137 download = root.find('download').attrib
1138 upload = root.find('upload').attrib
1139 # times = root.find('times').attrib
1140 client = root.find('client').attrib
1141
1142 except AttributeError:
1143 try:
1144 root = DOM.parseString(configxml)
1145 except ExpatError:
1146 e = get_exception()
1147 raise SpeedtestConfigError(
1148 'Malformed speedtest.net configuration: %s' % e
1149 )
1150 server_config = get_attributes_by_tag_name(root, 'server-config')
1151 download = get_attributes_by_tag_name(root, 'download')
1152 upload = get_attributes_by_tag_name(root, 'upload')
1153 # times = get_attributes_by_tag_name(root, 'times')
1154 client = get_attributes_by_tag_name(root, 'client')
1155
1156 ignore_servers = list(
1157 map(int, server_config['ignoreids'].split(','))
1158 )
1159
1160 ratio = int(upload['ratio'])
1161 upload_max = int(upload['maxchunkcount'])
1162 up_sizes = [32768, 65536, 131072, 262144, 524288, 1048576, 7340032]
1163 sizes = {
1164 'upload': up_sizes[ratio - 1:],
1165 'download': [350, 500, 750, 1000, 1500, 2000, 2500,
1166 3000, 3500, 4000]
1167 }
1168
1169 size_count = len(sizes['upload'])
1170
1171 upload_count = int(math.ceil(upload_max / size_count))
1172
1173 counts = {
1174 'upload': upload_count,
1175 'download': int(download['threadsperurl'])
1176 }
1177
1178 threads = {
1179 'upload': int(upload['threads']),
1180 'download': int(server_config['threadcount']) * 2
1181 }
1182
1183 length = {
1184 'upload': int(upload['testlength']),
1185 'download': int(download['testlength'])
1186 }
1187
1188 self.config.update({
1189 'client': client,
1190 'ignore_servers': ignore_servers,
1191 'sizes': sizes,
1192 'counts': counts,
1193 'threads': threads,
1194 'length': length,
1195 'upload_max': upload_count * size_count
1196 })
1197
1198 try:
1199 self.lat_lon = (float(client['lat']), float(client['lon']))
1200 except ValueError:
1201 raise SpeedtestConfigError(
1202 'Unknown location: lat=%r lon=%r' %
1203 (client.get('lat'), client.get('lon'))
1204 )
1205
1206 printer('Config:\n%r' % self.config, debug=True)
1207
1208 return self.config
1209
1210 def get_servers(self, servers=None, exclude=None):
1211 """Retrieve a the list of speedtest.net servers, optionally filtered
1212 to servers matching those specified in the ``servers`` argument
1213 """
1214 if servers is None:
1215 servers = []
1216
1217 if exclude is None:
1218 exclude = []
1219
1220 self.servers.clear()
1221
1222 for server_list in (servers, exclude):
1223 for i, s in enumerate(server_list):
1224 try:
1225 server_list[i] = int(s)
1226 except ValueError:
1227 raise InvalidServerIDType(
1228 '%s is an invalid server type, must be int' % s
1229 )
1230
1231 urls = [
1232 '://www.speedtest.net/speedtest-servers-static.php',
1233 'http://c.speedtest.net/speedtest-servers-static.php',
1234 '://www.speedtest.net/speedtest-servers.php',
1235 'http://c.speedtest.net/speedtest-servers.php',
1236 ]
1237
1238 headers = {}
1239 if gzip:
1240 headers['Accept-Encoding'] = 'gzip'
1241
1242 errors = []
1243 for url in urls:
1244 try:
1245 request = build_request(
1246 '%s?threads=%s' % (url,
1247 self.config['threads']['download']),
1248 headers=headers,
1249 secure=self._secure
1250 )
1251 uh, e = catch_request(request, opener=self._opener)
1252 if e:
1253 errors.append('%s' % e)
1254 raise ServersRetrievalError()
1255
1256 stream = get_response_stream(uh)
1257
1258 serversxml_list = []
1259 while 1:
1260 try:
1261 serversxml_list.append(stream.read(1024))
1262 except (OSError, EOFError):
1263 raise ServersRetrievalError(get_exception())
1264 if len(serversxml_list[-1]) == 0:
1265 break
1266
1267 stream.close()
1268 uh.close()
1269
1270 if int(uh.code) != 200:
1271 raise ServersRetrievalError()
1272
1273 serversxml = ''.encode().join(serversxml_list)
1274
1275 printer('Servers XML:\n%s' % serversxml, debug=True)
1276
1277 try:
1278 try:
1279 try:
1280 root = ET.fromstring(serversxml)
1281 except ET.ParseError:
1282 e = get_exception()
1283 raise SpeedtestServersError(
1284 'Malformed speedtest.net server list: %s' % e
1285 )
1286 elements = etree_iter(root, 'server')
1287 except AttributeError:
1288 try:
1289 root = DOM.parseString(serversxml)
1290 except ExpatError:
1291 e = get_exception()
1292 raise SpeedtestServersError(
1293 'Malformed speedtest.net server list: %s' % e
1294 )
1295 elements = root.getElementsByTagName('server')
1296 except (SyntaxError, xml.parsers.expat.ExpatError):
1297 raise ServersRetrievalError()
1298
1299 for server in elements:
1300 try:
1301 attrib = server.attrib
1302 except AttributeError:
1303 attrib = dict(list(server.attributes.items()))
1304
1305 if servers and int(attrib.get('id')) not in servers:
1306 continue
1307
1308 if (int(attrib.get('id')) in self.config['ignore_servers']
1309 or int(attrib.get('id')) in exclude):
1310 continue
1311
1312 try:
1313 d = distance(self.lat_lon,
1314 (float(attrib.get('lat')),
1315 float(attrib.get('lon'))))
1316 except Exception:
1317 continue
1318
1319 attrib['d'] = d
1320
1321 try:
1322 self.servers[d].append(attrib)
1323 except KeyError:
1324 self.servers[d] = [attrib]
1325
1326 break
1327
1328 except ServersRetrievalError:
1329 continue
1330
1331 if (servers or exclude) and not self.servers:
1332 raise NoMatchedServers()
1333
1334 return self.servers
1335
1336 def set_mini_server(self, server):
1337 """Instead of querying for a list of servers, set a link to a
1338 speedtest mini server
1339 """
1340
1341 urlparts = urlparse(server)
1342
1343 name, ext = os.path.splitext(urlparts[2])
1344 if ext:
1345 url = os.path.dirname(server)
1346 else:
1347 url = server
1348
1349 request = build_request(url)
1350 uh, e = catch_request(request, opener=self._opener)
1351 if e:
1352 raise SpeedtestMiniConnectFailure('Failed to connect to %s' %
1353 server)
1354 else:
1355 text = uh.read()
1356 uh.close()
1357
1358 extension = re.findall('upload_?[Ee]xtension: "([^"]+)"',
1359 text.decode())
1360 if not extension:
1361 for ext in ['php', 'asp', 'aspx', 'jsp']:
1362 try:
1363 f = self._opener.open(
1364 '%s/speedtest/upload.%s' % (url, ext)
1365 )
1366 except Exception:
1367 pass
1368 else:
1369 data = f.read().strip().decode()
1370 if (f.code == 200 and
1371 len(data.splitlines()) == 1 and
1372 re.match('size=[0-9]', data)):
1373 extension = [ext]
1374 break
1375 if not urlparts or not extension:
1376 raise InvalidSpeedtestMiniServer('Invalid Speedtest Mini Server: '
1377 '%s' % server)
1378
1379 self.servers = [{
1380 'sponsor': 'Speedtest Mini',
1381 'name': urlparts[1],
1382 'd': 0,
1383 'url': '%s/speedtest/upload.%s' % (url.rstrip('/'), extension[0]),
1384 'latency': 0,
1385 'id': 0
1386 }]
1387
1388 return self.servers
1389
1390 def get_closest_servers(self, limit=5):
1391 """Limit servers to the closest speedtest.net servers based on
1392 geographic distance
1393 """
1394
1395 if not self.servers:
1396 self.get_servers()
1397
1398 for d in sorted(self.servers.keys()):
1399 for s in self.servers[d]:
1400 self.closest.append(s)
1401 if len(self.closest) == limit:
1402 break
1403 else:
1404 continue
1405 break
1406
1407 printer('Closest Servers:\n%r' % self.closest, debug=True)
1408 return self.closest
1409
1410 def get_best_server(self, servers=None):
1411 """Perform a speedtest.net "ping" to determine which speedtest.net
1412 server has the lowest latency
1413 """
1414
1415 if not servers:
1416 if not self.closest:
1417 servers = self.get_closest_servers()
1418 servers = self.closest
1419
1420 if self._source_address:
1421 source_address_tuple = (self._source_address, 0)
1422 else:
1423 source_address_tuple = None
1424
1425 user_agent = build_user_agent()
1426
1427 results = {}
1428 for server in servers:
1429 cum = []
1430 url = os.path.dirname(server['url'])
1431 stamp = int(timeit.time.time() * 1000)
1432 latency_url = '%s/latency.txt?x=%s' % (url, stamp)
1433 for i in range(0, 3):
1434 this_latency_url = '%s.%s' % (latency_url, i)
1435 printer('%s %s' % ('GET', this_latency_url),
1436 debug=True)
1437 urlparts = urlparse(latency_url)
1438 try:
1439 if urlparts[0] == 'https':
1440 h = SpeedtestHTTPSConnection(
1441 urlparts[1],
1442 source_address=source_address_tuple
1443 )
1444 else:
1445 h = SpeedtestHTTPConnection(
1446 urlparts[1],
1447 source_address=source_address_tuple
1448 )
1449 headers = {'User-Agent': user_agent}
1450 path = '%s?%s' % (urlparts[2], urlparts[4])
1451 start = timeit.default_timer()
1452 h.request("GET", path, headers=headers)
1453 r = h.getresponse()
1454 total = (timeit.default_timer() - start)
1455 except HTTP_ERRORS:
1456 e = get_exception()
1457 printer('ERROR: %r' % e, debug=True)
1458 cum.append(3600)
1459 continue
1460
1461 text = r.read(9)
1462 if int(r.status) == 200 and text == 'test=test'.encode():
1463 cum.append(total)
1464 else:
1465 cum.append(3600)
1466 h.close()
1467
1468 avg = round((sum(cum) / 6) * 1000.0, 3)
1469 results[avg] = server
1470
1471 try:
1472 fastest = sorted(results.keys())[0]
1473 except IndexError:
1474 raise SpeedtestBestServerFailure('Unable to connect to servers to '
1475 'test latency.')
1476 best = results[fastest]
1477 best['latency'] = fastest
1478
1479 self.results.ping = fastest
1480 self.results.server = best
1481
1482 self._best.update(best)
1483 printer('Best Server:\n%r' % best, debug=True)
1484 return best
1485
1486 def download(self, callback=do_nothing, threads=None):
1487 """Test download speed against speedtest.net
1488
1489 A ``threads`` value of ``None`` will fall back to those dictated
1490 by the speedtest.net configuration
1491 """
1492
1493 urls = []
1494 for size in self.config['sizes']['download']:
1495 for _ in range(0, self.config['counts']['download']):
1496 urls.append('%s/random%sx%s.jpg' %
1497 (os.path.dirname(self.best['url']), size, size))
1498
1499 request_count = len(urls)
1500 requests = []
1501 for i, url in enumerate(urls):
1502 requests.append(
1503 build_request(url, bump=i, secure=self._secure)
1504 )
1505
1506 max_threads = threads or self.config['threads']['download']
1507 in_flight = {'threads': 0}
1508
1509 def producer(q, requests, request_count):
1510 for i, request in enumerate(requests):
1511 thread = HTTPDownloader(
1512 i,
1513 request,
1514 start,
1515 self.config['length']['download'],
1516 opener=self._opener,
1517 shutdown_event=self._shutdown_event
1518 )
1519 while in_flight['threads'] >= max_threads:
1520 timeit.time.sleep(0.001)
1521 thread.start()
1522 q.put(thread, True)
1523 in_flight['threads'] += 1
1524 callback(i, request_count, start=True)
1525
1526 finished = []
1527
1528 def consumer(q, request_count):
1529 _is_alive = thread_is_alive
1530 while len(finished) < request_count:
1531 thread = q.get(True)
1532 while _is_alive(thread):
1533 thread.join(timeout=0.001)
1534 in_flight['threads'] -= 1
1535 finished.append(sum(thread.result))
1536 callback(thread.i, request_count, end=True)
1537
1538 q = Queue(max_threads)
1539 prod_thread = threading.Thread(target=producer,
1540 args=(q, requests, request_count))
1541 cons_thread = threading.Thread(target=consumer,
1542 args=(q, request_count))
1543 start = timeit.default_timer()
1544 prod_thread.start()
1545 cons_thread.start()
1546 _is_alive = thread_is_alive
1547 while _is_alive(prod_thread):
1548 prod_thread.join(timeout=0.001)
1549 while _is_alive(cons_thread):
1550 cons_thread.join(timeout=0.001)
1551
1552 stop = timeit.default_timer()
1553 self.results.bytes_received = sum(finished)
1554 self.results.download = (
1555 (self.results.bytes_received / (stop - start)) * 8.0
1556 )
1557 if self.results.download > 100000:
1558 self.config['threads']['upload'] = 8
1559 return self.results.download
1560
1561 def upload(self, callback=do_nothing, pre_allocate=True, threads=None):
1562 """Test upload speed against speedtest.net
1563
1564 A ``threads`` value of ``None`` will fall back to those dictated
1565 by the speedtest.net configuration
1566 """
1567
1568 sizes = []
1569
1570 for size in self.config['sizes']['upload']:
1571 for _ in range(0, self.config['counts']['upload']):
1572 sizes.append(size)
1573
1574 # request_count = len(sizes)
1575 request_count = self.config['upload_max']
1576
1577 requests = []
1578 for i, size in enumerate(sizes):
1579 # We set ``0`` for ``start`` and handle setting the actual
1580 # ``start`` in ``HTTPUploader`` to get better measurements
1581 data = HTTPUploaderData(
1582 size,
1583 0,
1584 self.config['length']['upload'],
1585 shutdown_event=self._shutdown_event
1586 )
1587 if pre_allocate:
1588 data.pre_allocate()
1589
1590 headers = {'Content-length': size}
1591 requests.append(
1592 (
1593 build_request(self.best['url'], data, secure=self._secure,
1594 headers=headers),
1595 size
1596 )
1597 )
1598
1599 max_threads = threads or self.config['threads']['upload']
1600 in_flight = {'threads': 0}
1601
1602 def producer(q, requests, request_count):
1603 for i, request in enumerate(requests[:request_count]):
1604 thread = HTTPUploader(
1605 i,
1606 request[0],
1607 start,
1608 request[1],
1609 self.config['length']['upload'],
1610 opener=self._opener,
1611 shutdown_event=self._shutdown_event
1612 )
1613 while in_flight['threads'] >= max_threads:
1614 timeit.time.sleep(0.001)
1615 thread.start()
1616 q.put(thread, True)
1617 in_flight['threads'] += 1
1618 callback(i, request_count, start=True)
1619
1620 finished = []
1621
1622 def consumer(q, request_count):
1623 _is_alive = thread_is_alive
1624 while len(finished) < request_count:
1625 thread = q.get(True)
1626 while _is_alive(thread):
1627 thread.join(timeout=0.001)
1628 in_flight['threads'] -= 1
1629 finished.append(thread.result)
1630 callback(thread.i, request_count, end=True)
1631
1632 q = Queue(threads or self.config['threads']['upload'])
1633 prod_thread = threading.Thread(target=producer,
1634 args=(q, requests, request_count))
1635 cons_thread = threading.Thread(target=consumer,
1636 args=(q, request_count))
1637 start = timeit.default_timer()
1638 prod_thread.start()
1639 cons_thread.start()
1640 _is_alive = thread_is_alive
1641 while _is_alive(prod_thread):
1642 prod_thread.join(timeout=0.1)
1643 while _is_alive(cons_thread):
1644 cons_thread.join(timeout=0.1)
1645
1646 stop = timeit.default_timer()
1647 self.results.bytes_sent = sum(finished)
1648 self.results.upload = (
1649 (self.results.bytes_sent / (stop - start)) * 8.0
1650 )
1651 return self.results.upload
1652
1653
1654def ctrl_c(shutdown_event):
1655 """Catch Ctrl-C key sequence and set a SHUTDOWN_EVENT for our threaded
1656 operations
1657 """
1658 def inner(signum, frame):
1659 shutdown_event.set()
1660 printer('\nCancelling...', error=True)
1661 sys.exit(0)
1662 return inner
1663
1664
1665def version():
1666 """Print the version"""
1667
1668 printer('speedtest-cli %s' % __version__)
1669 printer('Python %s' % sys.version.replace('\n', ''))
1670 sys.exit(0)
1671
1672
1673def csv_header(delimiter=','):
1674 """Print the CSV Headers"""
1675
1676 printer(SpeedtestResults.csv_header(delimiter=delimiter))
1677 sys.exit(0)
1678
1679
1680def parse_args():
1681 """Function to handle building and parsing of command line arguments"""
1682 description = (
1683 'Command line interface for testing internet bandwidth using '
1684 'speedtest.net.\n'
1685 '------------------------------------------------------------'
1686 '--------------\n'
1687 'https://github.com/sivel/speedtest-cli')
1688
1689 parser = ArgParser(description=description)
1690 # Give optparse.OptionParser an `add_argument` method for
1691 # compatibility with argparse.ArgumentParser
1692 try:
1693 parser.add_argument = parser.add_option
1694 except AttributeError:
1695 pass
1696 parser.add_argument('--no-download', dest='download', default=True,
1697 action='store_const', const=False,
1698 help='Do not perform download test')
1699 parser.add_argument('--no-upload', dest='upload', default=True,
1700 action='store_const', const=False,
1701 help='Do not perform upload test')
1702 parser.add_argument('--single', default=False, action='store_true',
1703 help='Only use a single connection instead of '
1704 'multiple. This simulates a typical file '
1705 'transfer.')
1706 parser.add_argument('--bytes', dest='units', action='store_const',
1707 const=('byte', 8), default=('bit', 1),
1708 help='Display values in bytes instead of bits. Does '
1709 'not affect the image generated by --share, nor '
1710 'output from --json or --csv')
1711 parser.add_argument('--share', action='store_true',
1712 help='Generate and provide a URL to the speedtest.net '
1713 'share results image, not displayed with --csv')
1714 parser.add_argument('--simple', action='store_true', default=False,
1715 help='Suppress verbose output, only show basic '
1716 'information')
1717 parser.add_argument('--csv', action='store_true', default=False,
1718 help='Suppress verbose output, only show basic '
1719 'information in CSV format. Speeds listed in '
1720 'bit/s and not affected by --bytes')
1721 parser.add_argument('--csv-delimiter', default=',', type=PARSER_TYPE_STR,
1722 help='Single character delimiter to use in CSV '
1723 'output. Default ","')
1724 parser.add_argument('--csv-header', action='store_true', default=False,
1725 help='Print CSV headers')
1726 parser.add_argument('--json', action='store_true', default=False,
1727 help='Suppress verbose output, only show basic '
1728 'information in JSON format. Speeds listed in '
1729 'bit/s and not affected by --bytes')
1730 parser.add_argument('--list', action='store_true',
1731 help='Display a list of speedtest.net servers '
1732 'sorted by distance')
1733 parser.add_argument('--server', type=PARSER_TYPE_INT, action='append',
1734 help='Specify a server ID to test against. Can be '
1735 'supplied multiple times')
1736 parser.add_argument('--exclude', type=PARSER_TYPE_INT, action='append',
1737 help='Exclude a server from selection. Can be '
1738 'supplied multiple times')
1739 parser.add_argument('--mini', help='URL of the Speedtest Mini server')
1740 parser.add_argument('--source', help='Source IP address to bind to')
1741 parser.add_argument('--timeout', default=10, type=PARSER_TYPE_FLOAT,
1742 help='HTTP timeout in seconds. Default 10')
1743 parser.add_argument('--secure', action='store_true',
1744 help='Use HTTPS instead of HTTP when communicating '
1745 'with speedtest.net operated servers')
1746 parser.add_argument('--no-pre-allocate', dest='pre_allocate',
1747 action='store_const', default=True, const=False,
1748 help='Do not pre allocate upload data. Pre allocation '
1749 'is enabled by default to improve upload '
1750 'performance. To support systems with '
1751 'insufficient memory, use this option to avoid a '
1752 'MemoryError')
1753 parser.add_argument('--version', action='store_true',
1754 help='Show the version number and exit')
1755 parser.add_argument('--debug', action='store_true',
1756 help=ARG_SUPPRESS, default=ARG_SUPPRESS)
1757
1758 options = parser.parse_args()
1759 if isinstance(options, tuple):
1760 args = options[0]
1761 else:
1762 args = options
1763 return args
1764
1765
1766def validate_optional_args(args):
1767 """Check if an argument was provided that depends on a module that may
1768 not be part of the Python standard library.
1769
1770 If such an argument is supplied, and the module does not exist, exit
1771 with an error stating which module is missing.
1772 """
1773 optional_args = {
1774 'json': ('json/simplejson python module', json),
1775 'secure': ('SSL support', HTTPSConnection),
1776 }
1777
1778 for arg, info in optional_args.items():
1779 if getattr(args, arg, False) and info[1] is None:
1780 raise SystemExit('%s is not installed. --%s is '
1781 'unavailable' % (info[0], arg))
1782
1783
1784def printer(string, quiet=False, debug=False, error=False, **kwargs):
1785 """Helper function print a string with various features"""
1786
1787 if debug and not DEBUG:
1788 return
1789
1790 if debug:
1791 if sys.stdout.isatty():
1792 out = '\033[1;30mDEBUG: %s\033[0m' % string
1793 else:
1794 out = 'DEBUG: %s' % string
1795 else:
1796 out = string
1797
1798 if error:
1799 kwargs['file'] = sys.stderr
1800
1801 if not quiet:
1802 print_(out, **kwargs)
1803
1804
1805def shell():
1806 """Run the full speedtest.net test"""
1807
1808 global DEBUG
1809 shutdown_event = threading.Event()
1810
1811 signal.signal(signal.SIGINT, ctrl_c(shutdown_event))
1812
1813 args = parse_args()
1814
1815 # Print the version and exit
1816 if args.version:
1817 version()
1818
1819 if not args.download and not args.upload:
1820 raise SpeedtestCLIError('Cannot supply both --no-download and '
1821 '--no-upload')
1822
1823 if len(args.csv_delimiter) != 1:
1824 raise SpeedtestCLIError('--csv-delimiter must be a single character')
1825
1826 if args.csv_header:
1827 csv_header(args.csv_delimiter)
1828
1829 validate_optional_args(args)
1830
1831 debug = getattr(args, 'debug', False)
1832 if debug == 'SUPPRESSHELP':
1833 debug = False
1834 if debug:
1835 DEBUG = True
1836
1837 if args.simple or args.csv or args.json:
1838 quiet = True
1839 else:
1840 quiet = False
1841
1842 if args.csv or args.json:
1843 machine_format = True
1844 else:
1845 machine_format = False
1846
1847 # Don't set a callback if we are running quietly
1848 if quiet or debug:
1849 callback = do_nothing
1850 else:
1851 callback = print_dots(shutdown_event)
1852
1853 printer('Retrieving speedtest.net configuration...', quiet)
1854 try:
1855 speedtest = Speedtest(
1856 source_address=args.source,
1857 timeout=args.timeout,
1858 secure=args.secure
1859 )
1860 except (ConfigRetrievalError,) + HTTP_ERRORS:
1861 printer('Cannot retrieve speedtest configuration', error=True)
1862 raise SpeedtestCLIError(get_exception())
1863
1864 if args.list:
1865 try:
1866 speedtest.get_servers()
1867 except (ServersRetrievalError,) + HTTP_ERRORS:
1868 printer('Cannot retrieve speedtest server list', error=True)
1869 raise SpeedtestCLIError(get_exception())
1870
1871 for _, servers in sorted(speedtest.servers.items()):
1872 for server in servers:
1873 line = ('%(id)5s) %(sponsor)s (%(name)s, %(country)s) '
1874 '[%(d)0.2f km]' % server)
1875 try:
1876 printer(line)
1877 except IOError:
1878 e = get_exception()
1879 if e.errno != errno.EPIPE:
1880 raise
1881 sys.exit(0)
1882
1883 printer('Testing from %(isp)s (%(ip)s)...' % speedtest.config['client'],
1884 quiet)
1885
1886 if not args.mini:
1887 printer('Retrieving speedtest.net server list...', quiet)
1888 try:
1889 speedtest.get_servers(servers=args.server, exclude=args.exclude)
1890 except NoMatchedServers:
1891 raise SpeedtestCLIError(
1892 'No matched servers: %s' %
1893 ', '.join('%s' % s for s in args.server)
1894 )
1895 except (ServersRetrievalError,) + HTTP_ERRORS:
1896 printer('Cannot retrieve speedtest server list', error=True)
1897 raise SpeedtestCLIError(get_exception())
1898 except InvalidServerIDType:
1899 raise SpeedtestCLIError(
1900 '%s is an invalid server type, must '
1901 'be an int' % ', '.join('%s' % s for s in args.server)
1902 )
1903
1904 if args.server and len(args.server) == 1:
1905 printer('Retrieving information for the selected server...', quiet)
1906 else:
1907 printer('Selecting best server based on ping...', quiet)
1908 speedtest.get_best_server()
1909 elif args.mini:
1910 speedtest.get_best_server(speedtest.set_mini_server(args.mini))
1911
1912 results = speedtest.results
1913
1914 printer('Hosted by %(sponsor)s (%(name)s) [%(d)0.2f km]: '
1915 '%(latency)s ms' % results.server, quiet)
1916
1917 if args.download:
1918 printer('Testing download speed', quiet,
1919 end=('', '\n')[bool(debug)])
1920 speedtest.download(
1921 callback=callback,
1922 threads=(None, 1)[args.single]
1923 )
1924 printer('Download: %0.2f M%s/s' %
1925 ((results.download / 1000.0 / 1000.0) / args.units[1],
1926 args.units[0]),
1927 quiet)
1928 else:
1929 printer('Skipping download test', quiet)
1930
1931 if args.upload:
1932 printer('Testing upload speed', quiet,
1933 end=('', '\n')[bool(debug)])
1934 speedtest.upload(
1935 callback=callback,
1936 pre_allocate=args.pre_allocate,
1937 threads=(None, 1)[args.single]
1938 )
1939 printer('Upload: %0.2f M%s/s' %
1940 ((results.upload / 1000.0 / 1000.0) / args.units[1],
1941 args.units[0]),
1942 quiet)
1943 else:
1944 printer('Skipping upload test', quiet)
1945
1946 printer('Results:\n%r' % results.dict(), debug=True)
1947
1948 if not args.simple and args.share:
1949 results.share()
1950
1951 if args.simple:
1952 printer('Ping: %s ms\nDownload: %0.2f M%s/s\nUpload: %0.2f M%s/s' %
1953 (results.ping,
1954 (results.download / 1000.0 / 1000.0) / args.units[1],
1955 args.units[0],
1956 (results.upload / 1000.0 / 1000.0) / args.units[1],
1957 args.units[0]))
1958 elif args.csv:
1959 printer(results.csv(delimiter=args.csv_delimiter))
1960 elif args.json:
1961 printer(results.json())
1962
1963 if args.share and not machine_format:
1964 printer('Share results: %s' % results.share())
1965
1966
1967def main():
1968 try:
1969 shell()
1970 except KeyboardInterrupt:
1971 printer('\nCancelling...', error=True)
1972 except (SpeedtestException, SystemExit):
1973 e = get_exception()
1974 # Ignore a successful exit, or argparse exit
1975 if getattr(e, 'code', 1) not in (0, 2):
1976 msg = '%s' % e
1977 if not msg:
1978 msg = '%r' % e
1979 raise SystemExit('ERROR: %s' % msg)
1980
1981
1982if __name__ == '__main__':
1983 main()