YBDUbPPg

1#!/usr/bin/env python
2"""Extract list of URLs in a web page
3
4This program is part of "Dive Into Python", a free Python book for
5experienced programmers.  Visit http://diveintopython.org/ for the
6latest version.
7"""
8
9__author__ = "Mark Pilgrim (mark@diveintopython.org)"
10__version__ = "$Revision: 1.2 $"
11__date__ = "$Date: 2004/05/05 21:57:19 $"
12__copyright__ = "Copyright (c) 2001 Mark Pilgrim"
13__license__ = "Python"
14
15from sgmllib import SGMLParser
16import sys, urllib, getopt, socket, pdb
17from urlparse import urlparse
18try:
19        import psyco
20	psyco.full()
21except ImportError:
22        print "Psyco JIT Compiler (http://psyco.sf.net) not installed, performance may not be optimal"
23
24# defaults
25resolve_opt = 0
26file_opt = 0
27offline_opt = 0
28live_opt = 0
29name = []
30def showBanner():
31        print "Extract URLS From a Web Page\nUseage: list-url.py [-ahr] [-l url] [-o file] [-f filename]\n\t-l Online Page Retrive and Parse EX.. www.fark.com"
32	print "\t-o Offline Page Read and Parse EX.. ./index.html\n\t-r Resolve Hostnames to IP\n\t-f Output to a file EX.. Filename.txt\n\t-a Print the About\n\t-h Show this Menu"
33
34def about():
35        print "\n##########################################################"
36        print "#                   List URLS 3.0                        #"
37        print "#             Extract URLS from a web page               #"
38        print "#       Original code muts [AT] remote-exploit.com       #"
39        print "#         Feature update TheX1le [AT] gmail.com          #"
40        print "#                                                        #"
41        print "##########################################################\n"
42
43def parse(url):
44        lookup = urlparse(url) #I will note there is probably a faster way way to do this.
45        name.append(lookup.hostname)
46        if name.count(lookup.hostname) > 1:
47                name.remove(lookup.hostname)
48        else:
49                try:
50                        resolve(lookup.hostname,url)
51                except:
52                        pass
53
54def savefile(output):
55        FILE = open(filename, "a")
56        FILE.writelines(output + '\n')
57        FILE.close
58
59def resolve(name,url):
60        result = socket.getaddrinfo(name, None, 0, socket.SOCK_STREAM)
61        ip = [x[4][0] for x in result]
62        jip = ' '.join(ip)
63        result = name,jip,url
64        if file_opt == 1:
65                savefile(','.join(result))
66        else:
67                print ','.join(result)
68
69def live_operation(uri):
70        usock = urllib.urlopen(uri)
71        parser = URLLister()
72        parser.feed(usock.read())
73        parser.close()
74        usock.close()
75        return (parser)
76
77def url_prep(item): 
78	parse = urlparse(item)
79	item = ('http://' + parse.netloc + parse.path + parse.params + parse.query + parse.fragment)
80	return item	
81def offline_operation():
82        index = open(files, "r")
83        parser = URLLister()
84        parser.feed(index.read())
85        parser.close()
86        index.close()
87        return (parser)
88
89def press(parser):
90        if resolve_opt == 0:
91                for url in parser.urls:
92                        if parser.urls.count(url) > 1:
93                                parser.urls.remove(url)
94                        else:
95                               if file_opt == 1:
96                                        savefile(url)
97                               else:
98                                        print url
99
100        else:
101                for url in parser.urls:
102                        if parser.urls.count(url) > 1:
103                                parser.urls.remove(url)
104                        else:
105                                parse(url)
106class URLLister(SGMLParser):
107        def reset(self):
108                SGMLParser.reset(self)
109                self.urls = []
110
111        def start_a(self, attrs):
112                href = [v for k, v in attrs if k=='href']
113                if href:
114                        self.urls.extend(href)
115
116if __name__ == "__main__":
117        if len(sys.argv) <= 1:
118                about()
119                showBanner()
120                sys.exit(1)
121
122
123        try:
124                opts, args = getopt.getopt(sys.argv[1:], 'l:o:r,p:f:a,h')
125
126        except getopt.GetoptError, e:
127                print e
128
129        for o, a in opts:
130                if o == '-r':
131                        resolve_opt = 1
132
133                elif o == '-f':
134                        filename = a
135                        file_opt = 1
136
137                elif o == '-o':
138                        files = a
139                        offline_opt = 1
140
141                elif o == '-l':
142                        link = a
143                        test = urlparse(link)
144			live_opt = 1
145
146                elif o == '-a':
147                        about()
148                        sys.exit(0)
149                elif o == '-h':
150                        about()
151			showBanner()
152                        sys.exit(0)
153
154	if live_opt == 1 and offline_opt == 1:
155		print "You can not use Offline and Online at the same time"
156		sys.exit(1)
157	elif live_opt == 0 and offline_opt == 0:
158		print "You must select either Offine or Online to use this tool"
159		showBanner()
160	elif live_opt == 1:
161		return_var = url_prep(link)
162		return_var = live_operation(return_var)
163		return_var = press(return_var)
164	elif offline_opt == 1:
165		return_var = offline_operation()
166		return_var = press(return_var)