· 5 years ago · Jun 22, 2020, 08:46 AM
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# Copyright 2012 Matt Martz
4# All Rights Reserved.
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
18import os
19import re
20import csv
21import sys
22import math
23import errno
24import signal
25import socket
26import timeit
27import datetime
28import platform
29import threading
30import xml.parsers.expat
31
32try:
33 import gzip
34 GZIP_BASE = gzip.GzipFile
35except ImportError:
36 gzip = None
37 GZIP_BASE = object
38
39__version__ = '2.1.2'
40
41
42class FakeShutdownEvent(object):
43 """Class to fake a threading.Event.isSet so that users of this module
44 are not required to register their own threading.Event()
45 """
46
47 @staticmethod
48 def isSet():
49 "Dummy method to always return false"""
50 return False
51
52
53# Some global variables we use
54DEBUG = False
55_GLOBAL_DEFAULT_TIMEOUT = object()
56PY25PLUS = sys.version_info[:2] >= (2, 5)
57PY26PLUS = sys.version_info[:2] >= (2, 6)
58PY32PLUS = sys.version_info[:2] >= (3, 2)
59
60# Begin import game to handle Python 2 and Python 3
61try:
62 import json
63except ImportError:
64 try:
65 import simplejson as json
66 except ImportError:
67 json = None
68
69try:
70 import xml.etree.ElementTree as ET
71 try:
72 from xml.etree.ElementTree import _Element as ET_Element
73 except ImportError:
74 pass
75except ImportError:
76 from xml.dom import minidom as DOM
77 from xml.parsers.expat import ExpatError
78 ET = None
79
80try:
81 from urllib2 import (urlopen, Request, HTTPError, URLError,
82 AbstractHTTPHandler, ProxyHandler,
83 HTTPDefaultErrorHandler, HTTPRedirectHandler,
84 HTTPErrorProcessor, OpenerDirector)
85except ImportError:
86 from urllib.request import (urlopen, Request, HTTPError, URLError,
87 AbstractHTTPHandler, ProxyHandler,
88 HTTPDefaultErrorHandler, HTTPRedirectHandler,
89 HTTPErrorProcessor, OpenerDirector)
90
91try:
92 from httplib import HTTPConnection, BadStatusLine
93except ImportError:
94 from http.client import HTTPConnection, BadStatusLine
95
96try:
97 from httplib import HTTPSConnection
98except ImportError:
99 try:
100 from http.client import HTTPSConnection
101 except ImportError:
102 HTTPSConnection = None
103
104try:
105 from httplib import FakeSocket
106except ImportError:
107 FakeSocket = None
108
109try:
110 from Queue import Queue
111except ImportError:
112 from queue import Queue
113
114try:
115 from urlparse import urlparse
116except ImportError:
117 from urllib.parse import urlparse
118
119try:
120 from urlparse import parse_qs
121except ImportError:
122 try:
123 from urllib.parse import parse_qs
124 except ImportError:
125 from cgi import parse_qs
126
127try:
128 from hashlib import md5
129except ImportError:
130 from md5 import md5
131
132try:
133 from argparse import ArgumentParser as ArgParser
134 from argparse import SUPPRESS as ARG_SUPPRESS
135 PARSER_TYPE_INT = int
136 PARSER_TYPE_STR = str
137 PARSER_TYPE_FLOAT = float
138except ImportError:
139 from optparse import OptionParser as ArgParser
140 from optparse import SUPPRESS_HELP as ARG_SUPPRESS
141 PARSER_TYPE_INT = 'int'
142 PARSER_TYPE_STR = 'string'
143 PARSER_TYPE_FLOAT = 'float'
144
145try:
146 from cStringIO import StringIO
147 BytesIO = None
148except ImportError:
149 try:
150 from StringIO import StringIO
151 BytesIO = None
152 except ImportError:
153 from io import StringIO, BytesIO
154
155try:
156 import __builtin__
157except ImportError:
158 import builtins
159 from io import TextIOWrapper, FileIO
160
161 class _Py3Utf8Output(TextIOWrapper):
162 """UTF-8 encoded wrapper around stdout for py3, to override
163 ASCII stdout
164 """
165 def __init__(self, f, **kwargs):
166 buf = FileIO(f.fileno(), 'w')
167 super(_Py3Utf8Output, self).__init__(
168 buf,
169 encoding='utf8',
170 errors='strict'
171 )
172
173 def write(self, s):
174 super(_Py3Utf8Output, self).write(s)
175 self.flush()
176
177 _py3_print = getattr(builtins, 'print')
178 try:
179 _py3_utf8_stdout = _Py3Utf8Output(sys.stdout)
180 _py3_utf8_stderr = _Py3Utf8Output(sys.stderr)
181 except OSError:
182 # sys.stdout/sys.stderr is not a compatible stdout/stderr object
183 # just use it and hope things go ok
184 _py3_utf8_stdout = sys.stdout
185 _py3_utf8_stderr = sys.stderr
186
187 def to_utf8(v):
188 """No-op encode to utf-8 for py3"""
189 return v
190
191 def print_(*args, **kwargs):
192 """Wrapper function for py3 to print, with a utf-8 encoded stdout"""
193 if kwargs.get('file') == sys.stderr:
194 kwargs['file'] = _py3_utf8_stderr
195 else:
196 kwargs['file'] = kwargs.get('file', _py3_utf8_stdout)
197 _py3_print(*args, **kwargs)
198else:
199 del __builtin__
200
201 def to_utf8(v):
202 """Encode value to utf-8 if possible for py2"""
203 try:
204 return v.encode('utf8', 'strict')
205 except AttributeError:
206 return v
207
208 def print_(*args, **kwargs):
209 """The new-style print function for Python 2.4 and 2.5.
210
211 Taken from https://pypi.python.org/pypi/six/
212
213 Modified to set encoding to UTF-8 always, and to flush after write
214 """
215 fp = kwargs.pop("file", sys.stdout)
216 if fp is None:
217 return
218
219 def write(data):
220 if not isinstance(data, basestring):
221 data = str(data)
222 # If the file has an encoding, encode unicode with it.
223 encoding = 'utf8' # Always trust UTF-8 for output
224 if (isinstance(fp, file) and
225 isinstance(data, unicode) and
226 encoding is not None):
227 errors = getattr(fp, "errors", None)
228 if errors is None:
229 errors = "strict"
230 data = data.encode(encoding, errors)
231 fp.write(data)
232 fp.flush()
233 want_unicode = False
234 sep = kwargs.pop("sep", None)
235 if sep is not None:
236 if isinstance(sep, unicode):
237 want_unicode = True
238 elif not isinstance(sep, str):
239 raise TypeError("sep must be None or a string")
240 end = kwargs.pop("end", None)
241 if end is not None:
242 if isinstance(end, unicode):
243 want_unicode = True
244 elif not isinstance(end, str):
245 raise TypeError("end must be None or a string")
246 if kwargs:
247 raise TypeError("invalid keyword arguments to print()")
248 if not want_unicode:
249 for arg in args:
250 if isinstance(arg, unicode):
251 want_unicode = True
252 break
253 if want_unicode:
254 newline = unicode("\n")
255 space = unicode(" ")
256 else:
257 newline = "\n"
258 space = " "
259 if sep is None:
260 sep = space
261 if end is None:
262 end = newline
263 for i, arg in enumerate(args):
264 if i:
265 write(sep)
266 write(arg)
267 write(end)
268
269if PY32PLUS:
270 etree_iter = ET.Element.iter
271elif PY25PLUS:
272 etree_iter = ET_Element.getiterator
273
274if PY26PLUS:
275 thread_is_alive = threading.Thread.is_alive
276else:
277 thread_is_alive = threading.Thread.isAlive
278
279
280# Exception "constants" to support Python 2 through Python 3
281try:
282 import ssl
283 try:
284 CERT_ERROR = (ssl.CertificateError,)
285 except AttributeError:
286 CERT_ERROR = tuple()
287
288 HTTP_ERRORS = (
289 (HTTPError, URLError, socket.error, ssl.SSLError, BadStatusLine) +
290 CERT_ERROR
291 )
292except ImportError:
293 ssl = None
294 HTTP_ERRORS = (HTTPError, URLError, socket.error, BadStatusLine)
295
296
297class SpeedtestException(Exception):
298 """Base exception for this module"""
299
300
301class SpeedtestCLIError(SpeedtestException):
302 """Generic exception for raising errors during CLI operation"""
303
304
305class SpeedtestHTTPError(SpeedtestException):
306 """Base HTTP exception for this module"""
307
308
309class SpeedtestConfigError(SpeedtestException):
310 """Configuration XML is invalid"""
311
312
313class SpeedtestServersError(SpeedtestException):
314 """Servers XML is invalid"""
315
316
317class ConfigRetrievalError(SpeedtestHTTPError):
318 """Could not retrieve config.php"""
319
320
321class ServersRetrievalError(SpeedtestHTTPError):
322 """Could not retrieve speedtest-servers.php"""
323
324
325class InvalidServerIDType(SpeedtestException):
326 """Server ID used for filtering was not an integer"""
327
328
329class NoMatchedServers(SpeedtestException):
330 """No servers matched when filtering"""
331
332
333class SpeedtestMiniConnectFailure(SpeedtestException):
334 """Could not connect to the provided speedtest mini server"""
335
336
337class InvalidSpeedtestMiniServer(SpeedtestException):
338 """Server provided as a speedtest mini server does not actually appear
339 to be a speedtest mini server
340 """
341
342
343class ShareResultsConnectFailure(SpeedtestException):
344 """Could not connect to speedtest.net API to POST results"""
345
346
347class ShareResultsSubmitFailure(SpeedtestException):
348 """Unable to successfully POST results to speedtest.net API after
349 connection
350 """
351
352
353class SpeedtestUploadTimeout(SpeedtestException):
354 """testlength configuration reached during upload
355 Used to ensure the upload halts when no additional data should be sent
356 """
357
358
359class SpeedtestBestServerFailure(SpeedtestException):
360 """Unable to determine best server"""
361
362
363class SpeedtestMissingBestServer(SpeedtestException):
364 """get_best_server not called or not able to determine best server"""
365
366
367def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
368 source_address=None):
369 """Connect to *address* and return the socket object.
370
371 Convenience function. Connect to *address* (a 2-tuple ``(host,
372 port)``) and return the socket object. Passing the optional
373 *timeout* parameter will set the timeout on the socket instance
374 before attempting to connect. If no *timeout* is supplied, the
375 global default timeout setting returned by :func:`getdefaulttimeout`
376 is used. If *source_address* is set it must be a tuple of (host, port)
377 for the socket to bind as a source address before making the connection.
378 An host of '' or port 0 tells the OS to use the default.
379
380 Largely vendored from Python 2.7, modified to work with Python 2.4
381 """
382
383 host, port = address
384 err = None
385 for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
386 af, socktype, proto, canonname, sa = res
387 sock = None
388 try:
389 sock = socket.socket(af, socktype, proto)
390 if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
391 sock.settimeout(float(timeout))
392 if source_address:
393 sock.bind(source_address)
394 sock.connect(sa)
395 return sock
396
397 except socket.error:
398 err = get_exception()
399 if sock is not None:
400 sock.close()
401
402 if err is not None:
403 raise err
404 else:
405 raise socket.error("getaddrinfo returns an empty list")
406
407
408class SpeedtestHTTPConnection(HTTPConnection):
409 """Custom HTTPConnection to support source_address across
410 Python 2.4 - Python 3
411 """
412 def __init__(self, *args, **kwargs):
413 source_address = kwargs.pop('source_address', None)
414 timeout = kwargs.pop('timeout', 10)
415
416 self._tunnel_host = None
417
418 HTTPConnection.__init__(self, *args, **kwargs)
419
420 self.source_address = source_address
421 self.timeout = timeout
422
423 def connect(self):
424 """Connect to the host and port specified in __init__."""
425 try:
426 self.sock = socket.create_connection(
427 (self.host, self.port),
428 self.timeout,
429 self.source_address
430 )
431 except (AttributeError, TypeError):
432 self.sock = create_connection(
433 (self.host, self.port),
434 self.timeout,
435 self.source_address
436 )
437
438 if self._tunnel_host:
439 self._tunnel()
440
441
442if HTTPSConnection:
443 class SpeedtestHTTPSConnection(HTTPSConnection):
444 """Custom HTTPSConnection to support source_address across
445 Python 2.4 - Python 3
446 """
447 default_port = 443
448
449 def __init__(self, *args, **kwargs):
450 source_address = kwargs.pop('source_address', None)
451 timeout = kwargs.pop('timeout', 10)
452
453 self._tunnel_host = None
454
455 HTTPSConnection.__init__(self, *args, **kwargs)
456
457 self.timeout = timeout
458 self.source_address = source_address
459
460 def connect(self):
461 "Connect to a host on a given (SSL) port."
462 try:
463 self.sock = socket.create_connection(
464 (self.host, self.port),
465 self.timeout,
466 self.source_address
467 )
468 except (AttributeError, TypeError):
469 self.sock = create_connection(
470 (self.host, self.port),
471 self.timeout,
472 self.source_address
473 )
474
475 if self._tunnel_host:
476 self._tunnel()
477
478 if ssl:
479 try:
480 kwargs = {}
481 if hasattr(ssl, 'SSLContext'):
482 if self._tunnel_host:
483 kwargs['server_hostname'] = self._tunnel_host
484 else:
485 kwargs['server_hostname'] = self.host
486 self.sock = self._context.wrap_socket(self.sock, **kwargs)
487 except AttributeError:
488 self.sock = ssl.wrap_socket(self.sock)
489 try:
490 self.sock.server_hostname = self.host
491 except AttributeError:
492 pass
493 elif FakeSocket:
494 # Python 2.4/2.5 support
495 try:
496 self.sock = FakeSocket(self.sock, socket.ssl(self.sock))
497 except AttributeError:
498 raise SpeedtestException(
499 'This version of Python does not support HTTPS/SSL '
500 'functionality'
501 )
502 else:
503 raise SpeedtestException(
504 'This version of Python does not support HTTPS/SSL '
505 'functionality'
506 )
507
508
509def _build_connection(connection, source_address, timeout, context=None):
510 """Cross Python 2.4 - Python 3 callable to build an ``HTTPConnection`` or
511 ``HTTPSConnection`` with the args we need
512
513 Called from ``http(s)_open`` methods of ``SpeedtestHTTPHandler`` or
514 ``SpeedtestHTTPSHandler``
515 """
516 def inner(host, **kwargs):
517 kwargs.update({
518 'source_address': source_address,
519 'timeout': timeout
520 })
521 if context:
522 kwargs['context'] = context
523 return connection(host, **kwargs)
524 return inner
525
526
527class SpeedtestHTTPHandler(AbstractHTTPHandler):
528 """Custom ``HTTPHandler`` that can build a ``HTTPConnection`` with the
529 args we need for ``source_address`` and ``timeout``
530 """
531 def __init__(self, debuglevel=0, source_address=None, timeout=10):
532 AbstractHTTPHandler.__init__(self, debuglevel)
533 self.source_address = source_address
534 self.timeout = timeout
535
536 def http_open(self, req):
537 return self.do_open(
538 _build_connection(
539 SpeedtestHTTPConnection,
540 self.source_address,
541 self.timeout
542 ),
543 req
544 )
545
546 http_request = AbstractHTTPHandler.do_request_
547
548
549class SpeedtestHTTPSHandler(AbstractHTTPHandler):
550 """Custom ``HTTPSHandler`` that can build a ``HTTPSConnection`` with the
551 args we need for ``source_address`` and ``timeout``
552 """
553 def __init__(self, debuglevel=0, context=None, source_address=None,
554 timeout=10):
555 AbstractHTTPHandler.__init__(self, debuglevel)
556 self._context = context
557 self.source_address = source_address
558 self.timeout = timeout
559
560 def https_open(self, req):
561 return self.do_open(
562 _build_connection(
563 SpeedtestHTTPSConnection,
564 self.source_address,
565 self.timeout,
566 context=self._context,
567 ),
568 req
569 )
570
571 https_request = AbstractHTTPHandler.do_request_
572
573
574def build_opener(source_address=None, timeout=10):
575 """Function similar to ``urllib2.build_opener`` that will build
576 an ``OpenerDirector`` with the explicit handlers we want,
577 ``source_address`` for binding, ``timeout`` and our custom
578 `User-Agent`
579 """
580
581 printer('Timeout set to %d' % timeout, debug=True)
582
583 if source_address:
584 source_address_tuple = (source_address, 0)
585 printer('Binding to source address: %r' % (source_address_tuple,),
586 debug=True)
587 else:
588 source_address_tuple = None
589
590 handlers = [
591 ProxyHandler(),
592 SpeedtestHTTPHandler(source_address=source_address_tuple,
593 timeout=timeout),
594 SpeedtestHTTPSHandler(source_address=source_address_tuple,
595 timeout=timeout),
596 HTTPDefaultErrorHandler(),
597 HTTPRedirectHandler(),
598 HTTPErrorProcessor()
599 ]
600
601 opener = OpenerDirector()
602 opener.addheaders = [('User-agent', build_user_agent())]
603
604 for handler in handlers:
605 opener.add_handler(handler)
606
607 return opener
608
609
610class GzipDecodedResponse(GZIP_BASE):
611 """A file-like object to decode a response encoded with the gzip
612 method, as described in RFC 1952.
613
614 Largely copied from ``xmlrpclib``/``xmlrpc.client`` and modified
615 to work for py2.4-py3
616 """
617 def __init__(self, response):
618 # response doesn't support tell() and read(), required by
619 # GzipFile
620 if not gzip:
621 raise SpeedtestHTTPError('HTTP response body is gzip encoded, '
622 'but gzip support is not available')
623 IO = BytesIO or StringIO
624 self.io = IO()
625 while 1:
626 chunk = response.read(1024)
627 if len(chunk) == 0:
628 break
629 self.io.write(chunk)
630 self.io.seek(0)
631 gzip.GzipFile.__init__(self, mode='rb', fileobj=self.io)
632
633 def close(self):
634 try:
635 gzip.GzipFile.close(self)
636 finally:
637 self.io.close()
638
639
640def get_exception():
641 """Helper function to work with py2.4-py3 for getting the current
642 exception in a try/except block
643 """
644 return sys.exc_info()[1]
645
646
647def distance(origin, destination):
648 """Determine distance between 2 sets of [lat,lon] in km"""
649
650 lat1, lon1 = origin
651 lat2, lon2 = destination
652 radius = 6371 # km
653
654 dlat = math.radians(lat2 - lat1)
655 dlon = math.radians(lon2 - lon1)
656 a = (math.sin(dlat / 2) * math.sin(dlat / 2) +
657 math.cos(math.radians(lat1)) *
658 math.cos(math.radians(lat2)) * math.sin(dlon / 2) *
659 math.sin(dlon / 2))
660 c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
661 d = radius * c
662
663 return d
664
665
666def build_user_agent():
667 """Build a Mozilla/5.0 compatible User-Agent string"""
668
669 ua_tuple = (
670 'Mozilla/5.0',
671 '(%s; U; %s; en-us)' % (platform.platform(),
672 platform.architecture()[0]),
673 'Python/%s' % platform.python_version(),
674 '(KHTML, like Gecko)',
675 'speedtest-cli/%s' % __version__
676 )
677 user_agent = ' '.join(ua_tuple)
678 printer('User-Agent: %s' % user_agent, debug=True)
679 return user_agent
680
681
682def build_request(url, data=None, headers=None, bump='0', secure=False):
683 """Build a urllib2 request object
684
685 This function automatically adds a User-Agent header to all requests
686
687 """
688
689 if not headers:
690 headers = {}
691
692 if url[0] == ':':
693 scheme = ('http', 'https')[bool(secure)]
694 schemed_url = '%s%s' % (scheme, url)
695 else:
696 schemed_url = url
697
698 if '?' in url:
699 delim = '&'
700 else:
701 delim = '?'
702
703 # WHO YOU GONNA CALL? CACHE BUSTERS!
704 final_url = '%s%sx=%s.%s' % (schemed_url, delim,
705 int(timeit.time.time() * 1000),
706 bump)
707
708 headers.update({
709 'Cache-Control': 'no-cache',
710 })
711
712 printer('%s %s' % (('GET', 'POST')[bool(data)], final_url),
713 debug=True)
714
715 return Request(final_url, data=data, headers=headers)
716
717
718def catch_request(request, opener=None):
719 """Helper function to catch common exceptions encountered when
720 establishing a connection with a HTTP/HTTPS request
721
722 """
723
724 if opener:
725 _open = opener.open
726 else:
727 _open = urlopen
728
729 try:
730 uh = _open(request)
731 if request.get_full_url() != uh.geturl():
732 printer('Redirected to %s' % uh.geturl(), debug=True)
733 return uh, False
734 except HTTP_ERRORS:
735 e = get_exception()
736 return None, e
737
738
739def get_response_stream(response):
740 """Helper function to return either a Gzip reader if
741 ``Content-Encoding`` is ``gzip`` otherwise the response itself
742
743 """
744
745 try:
746 getheader = response.headers.getheader
747 except AttributeError:
748 getheader = response.getheader
749
750 if getheader('content-encoding') == 'gzip':
751 return GzipDecodedResponse(response)
752
753 return response
754
755
756def get_attributes_by_tag_name(dom, tag_name):
757 """Retrieve an attribute from an XML document and return it in a
758 consistent format
759
760 Only used with xml.dom.minidom, which is likely only to be used
761 with python versions older than 2.5
762 """
763 elem = dom.getElementsByTagName(tag_name)[0]
764 return dict(list(elem.attributes.items()))
765
766
767def print_dots(shutdown_event):
768 """Built in callback function used by Thread classes for printing
769 status
770 """
771 def inner(current, total, start=False, end=False):
772 if shutdown_event.isSet():
773 return
774
775 sys.stdout.write('.')
776 if current + 1 == total and end is True:
777 sys.stdout.write('\n')
778 sys.stdout.flush()
779 return inner
780
781
782def do_nothing(*args, **kwargs):
783 pass
784
785
786class HTTPDownloader(threading.Thread):
787 """Thread class for retrieving a URL"""
788
789 def __init__(self, i, request, start, timeout, opener=None,
790 shutdown_event=None):
791 threading.Thread.__init__(self)
792 self.request = request
793 self.result = [0]
794 self.starttime = start
795 self.timeout = timeout
796 self.i = i
797 if opener:
798 self._opener = opener.open
799 else:
800 self._opener = urlopen
801
802 if shutdown_event:
803 self._shutdown_event = shutdown_event
804 else:
805 self._shutdown_event = FakeShutdownEvent()
806
807 def run(self):
808 try:
809 if (timeit.default_timer() - self.starttime) <= self.timeout:
810 f = self._opener(self.request)
811 while (not self._shutdown_event.isSet() and
812 (timeit.default_timer() - self.starttime) <=
813 self.timeout):
814 self.result.append(len(f.read(10240)))
815 if self.result[-1] == 0:
816 break
817 f.close()
818 except IOError:
819 pass
820
821
822class HTTPUploaderData(object):
823 """File like object to improve cutting off the upload once the timeout
824 has been reached
825 """
826
827 def __init__(self, length, start, timeout, shutdown_event=None):
828 self.length = length
829 self.start = start
830 self.timeout = timeout
831
832 if shutdown_event:
833 self._shutdown_event = shutdown_event
834 else:
835 self._shutdown_event = FakeShutdownEvent()
836
837 self._data = None
838
839 self.total = [0]
840
841 def pre_allocate(self):
842 chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'
843 multiplier = int(round(int(self.length) / 36.0))
844 IO = BytesIO or StringIO
845 try:
846 self._data = IO(
847 ('content1=%s' %
848 (chars * multiplier)[0:int(self.length) - 9]
849 ).encode()
850 )
851 except MemoryError:
852 raise SpeedtestCLIError(
853 'Insufficient memory to pre-allocate upload data. Please '
854 'use --no-pre-allocate'
855 )
856
857 @property
858 def data(self):
859 if not self._data:
860 self.pre_allocate()
861 return self._data
862
863 def read(self, n=10240):
864 if ((timeit.default_timer() - self.start) <= self.timeout and
865 not self._shutdown_event.isSet()):
866 chunk = self.data.read(n)
867 self.total.append(len(chunk))
868 return chunk
869 else:
870 raise SpeedtestUploadTimeout()
871
872 def __len__(self):
873 return self.length
874
875
876class HTTPUploader(threading.Thread):
877 """Thread class for putting a URL"""
878
879 def __init__(self, i, request, start, size, timeout, opener=None,
880 shutdown_event=None):
881 threading.Thread.__init__(self)
882 self.request = request
883 self.request.data.start = self.starttime = start
884 self.size = size
885 self.result = None
886 self.timeout = timeout
887 self.i = i
888
889 if opener:
890 self._opener = opener.open
891 else:
892 self._opener = urlopen
893
894 if shutdown_event:
895 self._shutdown_event = shutdown_event
896 else:
897 self._shutdown_event = FakeShutdownEvent()
898
899 def run(self):
900 request = self.request
901 try:
902 if ((timeit.default_timer() - self.starttime) <= self.timeout and
903 not self._shutdown_event.isSet()):
904 try:
905 f = self._opener(request)
906 except TypeError:
907 # PY24 expects a string or buffer
908 # This also causes issues with Ctrl-C, but we will concede
909 # for the moment that Ctrl-C on PY24 isn't immediate
910 request = build_request(self.request.get_full_url(),
911 data=request.data.read(self.size))
912 f = self._opener(request)
913 f.read(11)
914 f.close()
915 self.result = sum(self.request.data.total)
916 else:
917 self.result = 0
918 except (IOError, SpeedtestUploadTimeout):
919 self.result = sum(self.request.data.total)
920
921
922class SpeedtestResults(object):
923 """Class for holding the results of a speedtest, including:
924
925 Download speed
926 Upload speed
927 Ping/Latency to test server
928 Data about server that the test was run against
929
930 Additionally this class can return a result data as a dictionary or CSV,
931 as well as submit a POST of the result data to the speedtest.net API
932 to get a share results image link.
933 """
934
935 def __init__(self, download=0, upload=0, ping=0, server=None, client=None,
936 opener=None, secure=False):
937 self.download = download
938 self.upload = upload
939 self.ping = ping
940 if server is None:
941 self.server = {}
942 else:
943 self.server = server
944 self.client = client or {}
945
946 self._share = None
947 self.timestamp = '%sZ' % datetime.datetime.utcnow().isoformat()
948 self.bytes_received = 0
949 self.bytes_sent = 0
950
951 if opener:
952 self._opener = opener
953 else:
954 self._opener = build_opener()
955
956 self._secure = secure
957
958 def __repr__(self):
959 return repr(self.dict())
960
961 def share(self):
962 """POST data to the speedtest.net API to obtain a share results
963 link
964 """
965
966 if self._share:
967 return self._share
968
969 download = int(round(self.download / 1000.0, 0))
970 ping = int(round(self.ping, 0))
971 upload = int(round(self.upload / 1000.0, 0))
972
973 # Build the request to send results back to speedtest.net
974 # We use a list instead of a dict because the API expects parameters
975 # in a certain order
976 api_data = [
977 'recommendedserverid=%s' % self.server['id'],
978 'ping=%s' % ping,
979 'screenresolution=',
980 'promo=',
981 'download=%s' % download,
982 'screendpi=',
983 'upload=%s' % upload,
984 'testmethod=http',
985 'hash=%s' % md5(('%s-%s-%s-%s' %
986 (ping, upload, download, '297aae72'))
987 .encode()).hexdigest(),
988 'touchscreen=none',
989 'startmode=pingselect',
990 'accuracy=1',
991 'bytesreceived=%s' % self.bytes_received,
992 'bytessent=%s' % self.bytes_sent,
993 'serverid=%s' % self.server['id'],
994 ]
995
996 headers = {'Referer': 'http://c.speedtest.net/flash/speedtest.swf'}
997 request = build_request('://www.speedtest.net/api/api.php',
998 data='&'.join(api_data).encode(),
999 headers=headers, secure=self._secure)
1000 f, e = catch_request(request, opener=self._opener)
1001 if e:
1002 raise ShareResultsConnectFailure(e)
1003
1004 response = f.read()
1005 code = f.code
1006 f.close()
1007
1008 if int(code) != 200:
1009 raise ShareResultsSubmitFailure('Could not submit results to '
1010 'speedtest.net')
1011
1012 qsargs = parse_qs(response.decode())
1013 resultid = qsargs.get('resultid')
1014 if not resultid or len(resultid) != 1:
1015 raise ShareResultsSubmitFailure('Could not submit results to '
1016 'speedtest.net')
1017
1018 self._share = 'http://www.speedtest.net/result/%s.png' % resultid[0]
1019
1020 return self._share
1021
1022 def dict(self):
1023 """Return dictionary of result data"""
1024
1025 return {
1026 'download': self.download,
1027 'upload': self.upload,
1028 'ping': self.ping,
1029 'server': self.server,
1030 'timestamp': self.timestamp,
1031 'bytes_sent': self.bytes_sent,
1032 'bytes_received': self.bytes_received,
1033 'share': self._share,
1034 'client': self.client,
1035 }
1036
1037 @staticmethod
1038 def csv_header(delimiter=','):
1039 """Return CSV Headers"""
1040
1041 row = ['Server ID', 'Sponsor', 'Server Name', 'Timestamp', 'Distance',
1042 'Ping', 'Download', 'Upload', 'Share', 'IP Address']
1043 out = StringIO()
1044 writer = csv.writer(out, delimiter=delimiter, lineterminator='')
1045 writer.writerow([to_utf8(v) for v in row])
1046 return out.getvalue()
1047
1048 def csv(self, delimiter=','):
1049 """Return data in CSV format"""
1050
1051 data = self.dict()
1052 out = StringIO()
1053 writer = csv.writer(out, delimiter=delimiter, lineterminator='')
1054 row = [data['server']['id'], data['server']['sponsor'],
1055 data['server']['name'], data['timestamp'],
1056 data['server']['d'], data['ping'], data['download'],
1057 data['upload'], self._share or '', self.client['ip']]
1058 writer.writerow([to_utf8(v) for v in row])
1059 return out.getvalue()
1060
1061 def json(self, pretty=False):
1062 """Return data in JSON format"""
1063
1064 kwargs = {}
1065 if pretty:
1066 kwargs.update({
1067 'indent': 4,
1068 'sort_keys': True
1069 })
1070 return json.dumps(self.dict(), **kwargs)
1071
1072
1073class Speedtest(object):
1074 """Class for performing standard speedtest.net testing operations"""
1075
1076 def __init__(self, config=None, source_address=None, timeout=10,
1077 secure=False, shutdown_event=None):
1078 self.config = {}
1079
1080 self._source_address = source_address
1081 self._timeout = timeout
1082 self._opener = build_opener(source_address, timeout)
1083
1084 self._secure = secure
1085
1086 if shutdown_event:
1087 self._shutdown_event = shutdown_event
1088 else:
1089 self._shutdown_event = FakeShutdownEvent()
1090
1091 self.get_config()
1092 if config is not None:
1093 self.config.update(config)
1094
1095 self.servers = {}
1096 self.closest = []
1097 self._best = {}
1098
1099 self.results = SpeedtestResults(
1100 client=self.config['client'],
1101 opener=self._opener,
1102 secure=secure,
1103 )
1104
1105 @property
1106 def best(self):
1107 if not self._best:
1108 self.get_best_server()
1109 return self._best
1110
1111 def get_config(self):
1112 """Download the speedtest.net configuration and return only the data
1113 we are interested in
1114 """
1115
1116 headers = {}
1117 if gzip:
1118 headers['Accept-Encoding'] = 'gzip'
1119 request = build_request('://www.speedtest.net/speedtest-config.php',
1120 headers=headers, secure=self._secure)
1121 uh, e = catch_request(request, opener=self._opener)
1122 if e:
1123 raise ConfigRetrievalError(e)
1124 configxml_list = []
1125
1126 stream = get_response_stream(uh)
1127
1128 while 1:
1129 try:
1130 configxml_list.append(stream.read(1024))
1131 except (OSError, EOFError):
1132 raise ConfigRetrievalError(get_exception())
1133 if len(configxml_list[-1]) == 0:
1134 break
1135 stream.close()
1136 uh.close()
1137
1138 if int(uh.code) != 200:
1139 return None
1140
1141 configxml = ''.encode().join(configxml_list)
1142
1143 printer('Config XML:\n%s' % configxml, debug=True)
1144
1145 try:
1146 try:
1147 root = ET.fromstring(configxml)
1148 except ET.ParseError:
1149 e = get_exception()
1150 raise SpeedtestConfigError(
1151 'Malformed speedtest.net configuration: %s' % e
1152 )
1153 server_config = root.find('server-config').attrib
1154 download = root.find('download').attrib
1155 upload = root.find('upload').attrib
1156 # times = root.find('times').attrib
1157 client = root.find('client').attrib
1158
1159 except AttributeError:
1160 try:
1161 root = DOM.parseString(configxml)
1162 except ExpatError:
1163 e = get_exception()
1164 raise SpeedtestConfigError(
1165 'Malformed speedtest.net configuration: %s' % e
1166 )
1167 server_config = get_attributes_by_tag_name(root, 'server-config')
1168 download = get_attributes_by_tag_name(root, 'download')
1169 upload = get_attributes_by_tag_name(root, 'upload')
1170 # times = get_attributes_by_tag_name(root, 'times')
1171 client = get_attributes_by_tag_name(root, 'client')
1172
1173 ignore_servers = list(
1174 map(int, server_config['ignoreids'].split(','))
1175 )
1176
1177 ratio = int(upload['ratio'])
1178 upload_max = int(upload['maxchunkcount'])
1179 up_sizes = [32768, 65536, 131072, 262144, 524288, 1048576, 7340032]
1180 sizes = {
1181 'upload': up_sizes[ratio - 1:],
1182 'download': [350, 500, 750, 1000, 1500, 2000, 2500,
1183 3000, 3500, 4000]
1184 }
1185
1186 size_count = len(sizes['upload'])
1187
1188 upload_count = int(math.ceil(upload_max / size_count))
1189
1190 counts = {
1191 'upload': upload_count,
1192 'download': int(download['threadsperurl'])
1193 }
1194
1195 threads = {
1196 'upload': int(upload['threads']),
1197 'download': int(server_config['threadcount']) * 2
1198 }
1199
1200 length = {
1201 'upload': int(upload['testlength']),
1202 'download': int(download['testlength'])
1203 }
1204
1205 self.config.update({
1206 'client': client,
1207 'ignore_servers': ignore_servers,
1208 'sizes': sizes,
1209 'counts': counts,
1210 'threads': threads,
1211 'length': length,
1212 'upload_max': upload_count * size_count
1213 })
1214
1215 try:
1216 self.lat_lon = (float(client['lat']), float(client['lon']))
1217 except ValueError:
1218 raise SpeedtestConfigError(
1219 'Unknown location: lat=%r lon=%r' %
1220 (client.get('lat'), client.get('lon'))
1221 )
1222
1223 printer('Config:\n%r' % self.config, debug=True)
1224
1225 return self.config
1226
1227 def get_servers(self, servers=None, exclude=None):
1228 """Retrieve a the list of speedtest.net servers, optionally filtered
1229 to servers matching those specified in the ``servers`` argument
1230 """
1231 if servers is None:
1232 servers = []
1233
1234 if exclude is None:
1235 exclude = []
1236
1237 self.servers.clear()
1238
1239 for server_list in (servers, exclude):
1240 for i, s in enumerate(server_list):
1241 try:
1242 server_list[i] = int(s)
1243 except ValueError:
1244 raise InvalidServerIDType(
1245 '%s is an invalid server type, must be int' % s
1246 )
1247
1248 urls = [
1249 '://www.speedtest.net/speedtest-servers-static.php',
1250 'http://c.speedtest.net/speedtest-servers-static.php',
1251 '://www.speedtest.net/speedtest-servers.php',
1252 'http://c.speedtest.net/speedtest-servers.php',
1253 ]
1254
1255 headers = {}
1256 if gzip:
1257 headers['Accept-Encoding'] = 'gzip'
1258
1259 errors = []
1260 for url in urls:
1261 try:
1262 request = build_request(
1263 '%s?threads=%s' % (url,
1264 self.config['threads']['download']),
1265 headers=headers,
1266 secure=self._secure
1267 )
1268 uh, e = catch_request(request, opener=self._opener)
1269 if e:
1270 errors.append('%s' % e)
1271 raise ServersRetrievalError()
1272
1273 stream = get_response_stream(uh)
1274
1275 serversxml_list = []
1276 while 1:
1277 try:
1278 serversxml_list.append(stream.read(1024))
1279 except (OSError, EOFError):
1280 raise ServersRetrievalError(get_exception())
1281 if len(serversxml_list[-1]) == 0:
1282 break
1283
1284 stream.close()
1285 uh.close()
1286
1287 if int(uh.code) != 200:
1288 raise ServersRetrievalError()
1289
1290 serversxml = ''.encode().join(serversxml_list)
1291
1292 printer('Servers XML:\n%s' % serversxml, debug=True)
1293
1294 try:
1295 try:
1296 try:
1297 root = ET.fromstring(serversxml)
1298 except ET.ParseError:
1299 e = get_exception()
1300 raise SpeedtestServersError(
1301 'Malformed speedtest.net server list: %s' % e
1302 )
1303 elements = etree_iter(root, 'server')
1304 except AttributeError:
1305 try:
1306 root = DOM.parseString(serversxml)
1307 except ExpatError:
1308 e = get_exception()
1309 raise SpeedtestServersError(
1310 'Malformed speedtest.net server list: %s' % e
1311 )
1312 elements = root.getElementsByTagName('server')
1313 except (SyntaxError, xml.parsers.expat.ExpatError):
1314 raise ServersRetrievalError()
1315
1316 for server in elements:
1317 try:
1318 attrib = server.attrib
1319 except AttributeError:
1320 attrib = dict(list(server.attributes.items()))
1321
1322 if servers and int(attrib.get('id')) not in servers:
1323 continue
1324
1325 if (int(attrib.get('id')) in self.config['ignore_servers']
1326 or int(attrib.get('id')) in exclude):
1327 continue
1328
1329 try:
1330 d = distance(self.lat_lon,
1331 (float(attrib.get('lat')),
1332 float(attrib.get('lon'))))
1333 except Exception:
1334 continue
1335
1336 attrib['d'] = d
1337
1338 try:
1339 self.servers[d].append(attrib)
1340 except KeyError:
1341 self.servers[d] = [attrib]
1342
1343 break
1344
1345 except ServersRetrievalError:
1346 continue
1347
1348 if (servers or exclude) and not self.servers:
1349 raise NoMatchedServers()
1350
1351 return self.servers
1352
1353 def set_mini_server(self, server):
1354 """Instead of querying for a list of servers, set a link to a
1355 speedtest mini server
1356 """
1357
1358 urlparts = urlparse(server)
1359
1360 name, ext = os.path.splitext(urlparts[2])
1361 if ext:
1362 url = os.path.dirname(server)
1363 else:
1364 url = server
1365
1366 request = build_request(url)
1367 uh, e = catch_request(request, opener=self._opener)
1368 if e:
1369 raise SpeedtestMiniConnectFailure('Failed to connect to %s' %
1370 server)
1371 else:
1372 text = uh.read()
1373 uh.close()
1374
1375 extension = re.findall('upload_?[Ee]xtension: "([^"]+)"',
1376 text.decode())
1377 if not extension:
1378 for ext in ['php', 'asp', 'aspx', 'jsp']:
1379 try:
1380 f = self._opener.open(
1381 '%s/speedtest/upload.%s' % (url, ext)
1382 )
1383 except Exception:
1384 pass
1385 else:
1386 data = f.read().strip().decode()
1387 if (f.code == 200 and
1388 len(data.splitlines()) == 1 and
1389 re.match('size=[0-9]', data)):
1390 extension = [ext]
1391 break
1392 if not urlparts or not extension:
1393 raise InvalidSpeedtestMiniServer('Invalid Speedtest Mini Server: '
1394 '%s' % server)
1395
1396 self.servers = [{
1397 'sponsor': 'Speedtest Mini',
1398 'name': urlparts[1],
1399 'd': 0,
1400 'url': '%s/speedtest/upload.%s' % (url.rstrip('/'), extension[0]),
1401 'latency': 0,
1402 'id': 0
1403 }]
1404
1405 return self.servers
1406
1407 def get_closest_servers(self, limit=5):
1408 """Limit servers to the closest speedtest.net servers based on
1409 geographic distance
1410 """
1411
1412 if not self.servers:
1413 self.get_servers()
1414
1415 for d in sorted(self.servers.keys()):
1416 for s in self.servers[d]:
1417 self.closest.append(s)
1418 if len(self.closest) == limit:
1419 break
1420 else:
1421 continue
1422 break
1423
1424 printer('Closest Servers:\n%r' % self.closest, debug=True)
1425 return self.closest
1426
1427 def get_best_server(self, servers=None):
1428 """Perform a speedtest.net "ping" to determine which speedtest.net
1429 server has the lowest latency
1430 """
1431
1432 if not servers:
1433 if not self.closest:
1434 servers = self.get_closest_servers()
1435 servers = self.closest
1436
1437 if self._source_address:
1438 source_address_tuple = (self._source_address, 0)
1439 else:
1440 source_address_tuple = None
1441
1442 user_agent = build_user_agent()
1443
1444 results = {}
1445 for server in servers:
1446 cum = []
1447 url = os.path.dirname(server['url'])
1448 stamp = int(timeit.time.time() * 1000)
1449 latency_url = '%s/latency.txt?x=%s' % (url, stamp)
1450 for i in range(0, 3):
1451 this_latency_url = '%s.%s' % (latency_url, i)
1452 printer('%s %s' % ('GET', this_latency_url),
1453 debug=True)
1454 urlparts = urlparse(latency_url)
1455 try:
1456 if urlparts[0] == 'https':
1457 h = SpeedtestHTTPSConnection(
1458 urlparts[1],
1459 source_address=source_address_tuple
1460 )
1461 else:
1462 h = SpeedtestHTTPConnection(
1463 urlparts[1],
1464 source_address=source_address_tuple
1465 )
1466 headers = {'User-Agent': user_agent}
1467 path = '%s?%s' % (urlparts[2], urlparts[4])
1468 start = timeit.default_timer()
1469 h.request("GET", path, headers=headers)
1470 r = h.getresponse()
1471 total = (timeit.default_timer() - start)
1472 except HTTP_ERRORS:
1473 e = get_exception()
1474 printer('ERROR: %r' % e, debug=True)
1475 cum.append(3600)
1476 continue
1477
1478 text = r.read(9)
1479 if int(r.status) == 200 and text == 'test=test'.encode():
1480 cum.append(total)
1481 else:
1482 cum.append(3600)
1483 h.close()
1484
1485 avg = round((sum(cum) / 6) * 1000.0, 3)
1486 results[avg] = server
1487
1488 try:
1489 fastest = sorted(results.keys())[0]
1490 except IndexError:
1491 raise SpeedtestBestServerFailure('Unable to connect to servers to '
1492 'test latency.')
1493 best = results[fastest]
1494 best['latency'] = fastest
1495
1496 self.results.ping = fastest
1497 self.results.server = best
1498
1499 self._best.update(best)
1500 printer('Best Server:\n%r' % best, debug=True)
1501 return best
1502
1503 def download(self, callback=do_nothing, threads=None):
1504 """Test download speed against speedtest.net
1505
1506 A ``threads`` value of ``None`` will fall back to those dictated
1507 by the speedtest.net configuration
1508 """
1509
1510 urls = []
1511 for size in self.config['sizes']['download']:
1512 for _ in range(0, self.config['counts']['download']):
1513 urls.append('%s/random%sx%s.jpg' %
1514 (os.path.dirname(self.best['url']), size, size))
1515
1516 request_count = len(urls)
1517 requests = []
1518 for i, url in enumerate(urls):
1519 requests.append(
1520 build_request(url, bump=i, secure=self._secure)
1521 )
1522
1523 max_threads = threads or self.config['threads']['download']
1524 in_flight = {'threads': 0}
1525
1526 def producer(q, requests, request_count):
1527 for i, request in enumerate(requests):
1528 thread = HTTPDownloader(
1529 i,
1530 request,
1531 start,
1532 self.config['length']['download'],
1533 opener=self._opener,
1534 shutdown_event=self._shutdown_event
1535 )
1536 while in_flight['threads'] >= max_threads:
1537 timeit.time.sleep(0.001)
1538 thread.start()
1539 q.put(thread, True)
1540 in_flight['threads'] += 1
1541 callback(i, request_count, start=True)
1542
1543 finished = []
1544
1545 def consumer(q, request_count):
1546 _is_alive = thread_is_alive
1547 while len(finished) < request_count:
1548 thread = q.get(True)
1549 while _is_alive(thread):
1550 thread.join(timeout=0.001)
1551 in_flight['threads'] -= 1
1552 finished.append(sum(thread.result))
1553 callback(thread.i, request_count, end=True)
1554
1555 q = Queue(max_threads)
1556 prod_thread = threading.Thread(target=producer,
1557 args=(q, requests, request_count))
1558 cons_thread = threading.Thread(target=consumer,
1559 args=(q, request_count))
1560 start = timeit.default_timer()
1561 prod_thread.start()
1562 cons_thread.start()
1563 _is_alive = thread_is_alive
1564 while _is_alive(prod_thread):
1565 prod_thread.join(timeout=0.001)
1566 while _is_alive(cons_thread):
1567 cons_thread.join(timeout=0.001)
1568
1569 stop = timeit.default_timer()
1570 self.results.bytes_received = sum(finished)
1571 self.results.download = (
1572 (self.results.bytes_received / (stop - start)) * 8.0
1573 )
1574 if self.results.download > 100000:
1575 self.config['threads']['upload'] = 8
1576 return self.results.download
1577
1578 def upload(self, callback=do_nothing, pre_allocate=True, threads=None):
1579 """Test upload speed against speedtest.net
1580
1581 A ``threads`` value of ``None`` will fall back to those dictated
1582 by the speedtest.net configuration
1583 """
1584
1585 sizes = []
1586
1587 for size in self.config['sizes']['upload']:
1588 for _ in range(0, self.config['counts']['upload']):
1589 sizes.append(size)
1590
1591 # request_count = len(sizes)
1592 request_count = self.config['upload_max']
1593
1594 requests = []
1595 for i, size in enumerate(sizes):
1596 # We set ``0`` for ``start`` and handle setting the actual
1597 # ``start`` in ``HTTPUploader`` to get better measurements
1598 data = HTTPUploaderData(
1599 size,
1600 0,
1601 self.config['length']['upload'],
1602 shutdown_event=self._shutdown_event
1603 )
1604 if pre_allocate:
1605 data.pre_allocate()
1606
1607 headers = {'Content-length': size}
1608 requests.append(
1609 (
1610 build_request(self.best['url'], data, secure=self._secure,
1611 headers=headers),
1612 size
1613 )
1614 )
1615
1616 max_threads = threads or self.config['threads']['upload']
1617 in_flight = {'threads': 0}
1618
1619 def producer(q, requests, request_count):
1620 for i, request in enumerate(requests[:request_count]):
1621 thread = HTTPUploader(
1622 i,
1623 request[0],
1624 start,
1625 request[1],
1626 self.config['length']['upload'],
1627 opener=self._opener,
1628 shutdown_event=self._shutdown_event
1629 )
1630 while in_flight['threads'] >= max_threads:
1631 timeit.time.sleep(0.001)
1632 thread.start()
1633 q.put(thread, True)
1634 in_flight['threads'] += 1
1635 callback(i, request_count, start=True)
1636
1637 finished = []
1638
1639 def consumer(q, request_count):
1640 _is_alive = thread_is_alive
1641 while len(finished) < request_count:
1642 thread = q.get(True)
1643 while _is_alive(thread):
1644 thread.join(timeout=0.001)
1645 in_flight['threads'] -= 1
1646 finished.append(thread.result)
1647 callback(thread.i, request_count, end=True)
1648
1649 q = Queue(threads or self.config['threads']['upload'])
1650 prod_thread = threading.Thread(target=producer,
1651 args=(q, requests, request_count))
1652 cons_thread = threading.Thread(target=consumer,
1653 args=(q, request_count))
1654 start = timeit.default_timer()
1655 prod_thread.start()
1656 cons_thread.start()
1657 _is_alive = thread_is_alive
1658 while _is_alive(prod_thread):
1659 prod_thread.join(timeout=0.1)
1660 while _is_alive(cons_thread):
1661 cons_thread.join(timeout=0.1)
1662
1663 stop = timeit.default_timer()
1664 self.results.bytes_sent = sum(finished)
1665 self.results.upload = (
1666 (self.results.bytes_sent / (stop - start)) * 8.0
1667 )
1668 return self.results.upload
1669
1670
1671def ctrl_c(shutdown_event):
1672 """Catch Ctrl-C key sequence and set a SHUTDOWN_EVENT for our threaded
1673 operations
1674 """
1675 def inner(signum, frame):
1676 shutdown_event.set()
1677 printer('\nCancelling...', error=True)
1678 sys.exit(0)
1679 return inner
1680
1681
1682def version():
1683 """Print the version"""
1684
1685 printer('speedtest-cli %s' % __version__)
1686 printer('Python %s' % sys.version.replace('\n', ''))
1687 sys.exit(0)
1688
1689
1690def csv_header(delimiter=','):
1691 """Print the CSV Headers"""
1692
1693 printer(SpeedtestResults.csv_header(delimiter=delimiter))
1694 sys.exit(0)
1695
1696
1697def parse_args():
1698 """Function to handle building and parsing of command line arguments"""
1699 description = (
1700 'Command line interface for testing internet bandwidth using '
1701 'speedtest.net.\n'
1702 '------------------------------------------------------------'
1703 '--------------\n'
1704 'https://github.com/sivel/speedtest-cli')
1705
1706 parser = ArgParser(description=description)
1707 # Give optparse.OptionParser an `add_argument` method for
1708 # compatibility with argparse.ArgumentParser
1709 try:
1710 parser.add_argument = parser.add_option
1711 except AttributeError:
1712 pass
1713 parser.add_argument('--no-download', dest='download', default=True,
1714 action='store_const', const=False,
1715 help='Do not perform download test')
1716 parser.add_argument('--no-upload', dest='upload', default=True,
1717 action='store_const', const=False,
1718 help='Do not perform upload test')
1719 parser.add_argument('--single', default=False, action='store_true',
1720 help='Only use a single connection instead of '
1721 'multiple. This simulates a typical file '
1722 'transfer.')
1723 parser.add_argument('--bytes', dest='units', action='store_const',
1724 const=('byte', 8), default=('bit', 1),
1725 help='Display values in bytes instead of bits. Does '
1726 'not affect the image generated by --share, nor '
1727 'output from --json or --csv')
1728 parser.add_argument('--share', action='store_true',
1729 help='Generate and provide a URL to the speedtest.net '
1730 'share results image, not displayed with --csv')
1731 parser.add_argument('--simple', action='store_true', default=False,
1732 help='Suppress verbose output, only show basic '
1733 'information')
1734 parser.add_argument('--csv', action='store_true', default=False,
1735 help='Suppress verbose output, only show basic '
1736 'information in CSV format. Speeds listed in '
1737 'bit/s and not affected by --bytes')
1738 parser.add_argument('--csv-delimiter', default=',', type=PARSER_TYPE_STR,
1739 help='Single character delimiter to use in CSV '
1740 'output. Default ","')
1741 parser.add_argument('--csv-header', action='store_true', default=False,
1742 help='Print CSV headers')
1743 parser.add_argument('--json', action='store_true', default=False,
1744 help='Suppress verbose output, only show basic '
1745 'information in JSON format. Speeds listed in '
1746 'bit/s and not affected by --bytes')
1747 parser.add_argument('--list', action='store_true',
1748 help='Display a list of speedtest.net servers '
1749 'sorted by distance')
1750 parser.add_argument('--server', type=PARSER_TYPE_INT, action='append',
1751 help='Specify a server ID to test against. Can be '
1752 'supplied multiple times')
1753 parser.add_argument('--exclude', type=PARSER_TYPE_INT, action='append',
1754 help='Exclude a server from selection. Can be '
1755 'supplied multiple times')
1756 parser.add_argument('--mini', help='URL of the Speedtest Mini server')
1757 parser.add_argument('--source', help='Source IP address to bind to')
1758 parser.add_argument('--timeout', default=10, type=PARSER_TYPE_FLOAT,
1759 help='HTTP timeout in seconds. Default 10')
1760 parser.add_argument('--secure', action='store_true',
1761 help='Use HTTPS instead of HTTP when communicating '
1762 'with speedtest.net operated servers')
1763 parser.add_argument('--no-pre-allocate', dest='pre_allocate',
1764 action='store_const', default=True, const=False,
1765 help='Do not pre allocate upload data. Pre allocation '
1766 'is enabled by default to improve upload '
1767 'performance. To support systems with '
1768 'insufficient memory, use this option to avoid a '
1769 'MemoryError')
1770 parser.add_argument('--version', action='store_true',
1771 help='Show the version number and exit')
1772 parser.add_argument('--debug', action='store_true',
1773 help=ARG_SUPPRESS, default=ARG_SUPPRESS)
1774
1775 options = parser.parse_args()
1776 if isinstance(options, tuple):
1777 args = options[0]
1778 else:
1779 args = options
1780 return args
1781
1782
1783def validate_optional_args(args):
1784 """Check if an argument was provided that depends on a module that may
1785 not be part of the Python standard library.
1786
1787 If such an argument is supplied, and the module does not exist, exit
1788 with an error stating which module is missing.
1789 """
1790 optional_args = {
1791 'json': ('json/simplejson python module', json),
1792 'secure': ('SSL support', HTTPSConnection),
1793 }
1794
1795 for arg, info in optional_args.items():
1796 if getattr(args, arg, False) and info[1] is None:
1797 raise SystemExit('%s is not installed. --%s is '
1798 'unavailable' % (info[0], arg))
1799
1800
1801def printer(string, quiet=False, debug=False, error=False, **kwargs):
1802 """Helper function print a string with various features"""
1803
1804 if debug and not DEBUG:
1805 return
1806
1807 if debug:
1808 if sys.stdout.isatty():
1809 out = '\033[1;30mDEBUG: %s\033[0m' % string
1810 else:
1811 out = 'DEBUG: %s' % string
1812 else:
1813 out = string
1814
1815 if error:
1816 kwargs['file'] = sys.stderr
1817
1818 if not quiet:
1819 print_(out, **kwargs)
1820
1821
1822def shell():
1823 """Run the full speedtest.net test"""
1824
1825 global DEBUG
1826 shutdown_event = threading.Event()
1827
1828 signal.signal(signal.SIGINT, ctrl_c(shutdown_event))
1829
1830 args = parse_args()
1831
1832 # Print the version and exit
1833 if args.version:
1834 version()
1835
1836 if not args.download and not args.upload:
1837 raise SpeedtestCLIError('Cannot supply both --no-download and '
1838 '--no-upload')
1839
1840 if len(args.csv_delimiter) != 1:
1841 raise SpeedtestCLIError('--csv-delimiter must be a single character')
1842
1843 if args.csv_header:
1844 csv_header(args.csv_delimiter)
1845
1846 validate_optional_args(args)
1847
1848 debug = getattr(args, 'debug', False)
1849 if debug == 'SUPPRESSHELP':
1850 debug = False
1851 if debug:
1852 DEBUG = True
1853
1854 if args.simple or args.csv or args.json:
1855 quiet = True
1856 else:
1857 quiet = False
1858
1859 if args.csv or args.json:
1860 machine_format = True
1861 else:
1862 machine_format = False
1863
1864 # Don't set a callback if we are running quietly
1865 if quiet or debug:
1866 callback = do_nothing
1867 else:
1868 callback = print_dots(shutdown_event)
1869
1870 printer('Retrieving speedtest.net configuration...', quiet)
1871 try:
1872 speedtest = Speedtest(
1873 source_address=args.source,
1874 timeout=args.timeout,
1875 secure=args.secure
1876 )
1877 except (ConfigRetrievalError,) + HTTP_ERRORS:
1878 printer('Cannot retrieve speedtest configuration', error=True)
1879 raise SpeedtestCLIError(get_exception())
1880
1881 if args.list:
1882 try:
1883 speedtest.get_servers()
1884 except (ServersRetrievalError,) + HTTP_ERRORS:
1885 printer('Cannot retrieve speedtest server list', error=True)
1886 raise SpeedtestCLIError(get_exception())
1887
1888 for _, servers in sorted(speedtest.servers.items()):
1889 for server in servers:
1890 line = ('%(id)5s) %(sponsor)s (%(name)s, %(country)s) '
1891 '[%(d)0.2f km]' % server)
1892 try:
1893 printer(line)
1894 except IOError:
1895 e = get_exception()
1896 if e.errno != errno.EPIPE:
1897 raise
1898 sys.exit(0)
1899
1900 printer('Testing from %(isp)s (%(ip)s)...' % speedtest.config['client'],
1901 quiet)
1902
1903 if not args.mini:
1904 printer('Retrieving speedtest.net server list...', quiet)
1905 try:
1906 speedtest.get_servers(servers=args.server, exclude=args.exclude)
1907 except NoMatchedServers:
1908 raise SpeedtestCLIError(
1909 'No matched servers: %s' %
1910 ', '.join('%s' % s for s in args.server)
1911 )
1912 except (ServersRetrievalError,) + HTTP_ERRORS:
1913 printer('Cannot retrieve speedtest server list', error=True)
1914 raise SpeedtestCLIError(get_exception())
1915 except InvalidServerIDType:
1916 raise SpeedtestCLIError(
1917 '%s is an invalid server type, must '
1918 'be an int' % ', '.join('%s' % s for s in args.server)
1919 )
1920
1921 if args.server and len(args.server) == 1:
1922 printer('Retrieving information for the selected server...', quiet)
1923 else:
1924 printer('Selecting best server based on ping...', quiet)
1925 speedtest.get_best_server()
1926 elif args.mini:
1927 speedtest.get_best_server(speedtest.set_mini_server(args.mini))
1928
1929 results = speedtest.results
1930
1931 printer('Hosted by %(sponsor)s (%(name)s) [%(d)0.2f km]: '
1932 '%(latency)s ms' % results.server, quiet)
1933
1934 if args.download:
1935 printer('Testing download speed', quiet,
1936 end=('', '\n')[bool(debug)])
1937 speedtest.download(
1938 callback=callback,
1939 threads=(None, 1)[args.single]
1940 )
1941 printer('Download: %0.2f M%s/s' %
1942 ((results.download / 1000.0 / 1000.0) / args.units[1],
1943 args.units[0]),
1944 quiet)
1945 else:
1946 printer('Skipping download test', quiet)
1947
1948 if args.upload:
1949 printer('Testing upload speed', quiet,
1950 end=('', '\n')[bool(debug)])
1951 speedtest.upload(
1952 callback=callback,
1953 pre_allocate=args.pre_allocate,
1954 threads=(None, 1)[args.single]
1955 )
1956 printer('Upload: %0.2f M%s/s' %
1957 ((results.upload / 1000.0 / 1000.0) / args.units[1],
1958 args.units[0]),
1959 quiet)
1960 else:
1961 printer('Skipping upload test', quiet)
1962
1963 printer('Results:\n%r' % results.dict(), debug=True)
1964
1965 if not args.simple and args.share:
1966 results.share()
1967
1968 if args.simple:
1969 printer('Ping: %s ms\nDownload: %0.2f M%s/s\nUpload: %0.2f M%s/s' %
1970 (results.ping,
1971 (results.download / 1000.0 / 1000.0) / args.units[1],
1972 args.units[0],
1973 (results.upload / 1000.0 / 1000.0) / args.units[1],
1974 args.units[0]))
1975 elif args.csv:
1976 printer(results.csv(delimiter=args.csv_delimiter))
1977 elif args.json:
1978 printer(results.json())
1979
1980 if args.share and not machine_format:
1981 printer('Share results: %s' % results.share())
1982
1983
1984def main():
1985 try:
1986 shell()
1987 except KeyboardInterrupt:
1988 printer('\nCancelling...', error=True)
1989 except (SpeedtestException, SystemExit):
1990 e = get_exception()
1991 # Ignore a successful exit, or argparse exit
1992 if getattr(e, 'code', 1) not in (0, 2):
1993 msg = '%s' % e
1994 if not msg:
1995 msg = '%r' % e
1996 raise SystemExit('ERROR: %s' % msg)
1997
1998
1999if __name__ == '__main__':
2000 main()