· 9 years ago · Mar 13, 2017, 12:20 PM
1---Implements the HTTP client protocol in a standard form that Nmap scripts can
2-- take advantage of.
3--
4-- Because HTTP has so many uses, there are a number of interfaces to this library.
5-- The most obvious and common ones are simply <code>get</code>, <code>post</code>,
6-- and <code>head</code>; or, if more control is required, <code>generic_request</code>
7-- can be used. These functions do what one would expect. The <code>get_url</code>
8-- helper function can be used to parse and retrieve a full URL.
9--
10-- HTTPS support is transparent. The library uses <code>comm.tryssl</code> to
11-- determine whether SSL is required for a request.
12--
13-- These functions return a table of values, including:
14-- * <code>status-line</code> - A string representing the status, such as "HTTP/1.1 200 OK". In case of an error, a description will be provided in this line.
15-- * <code>status</code>: The HTTP status value; for example, "200". If an error occurs during a request, then this value is going to be nil.
16-- * <code>header</code> - An associative array representing the header. Keys are all lowercase, and standard headers, such as 'date', 'content-length', etc. will typically be present.
17-- * <code>rawheader</code> - A numbered array of the headers, exactly as the server sent them. While header['content-type'] might be 'text/html', rawheader[3] might be 'Content-type: text/html'.
18-- * <code>cookies</code> - A numbered array of the cookies the server sent. Each cookie is a table with the following keys: <code>name</code>, <code>value</code>, <code>path</code>, <code>domain</code>, and <code>expires</code>.
19-- * <code>body</code> - The full body, as returned by the server.
20--
21-- If a script is planning on making a lot of requests, the pipelining functions can
22-- be helpful. <code>pipeline_add</code> queues requests in a table, and
23-- <code>pipeline_go</code> performs the requests, returning the results as an array,
24-- with the responses in the same order as the queries were added. As a simple example:
25--<code>
26-- -- Start by defining the 'all' variable as nil
27-- local all = nil
28--
29-- -- Add two 'GET' requests and one 'HEAD' to the queue. These requests are not performed
30-- -- yet. The second parameter represents the 'options' table, which we don't need.
31-- all = http.pipeline_add('/book', nil, all)
32-- all = http.pipeline_add('/test', nil, all)
33-- all = http.pipeline_add('/monkeys', nil, all, 'HEAD')
34--
35-- -- Perform all three requests as parallel as Nmap is able to
36-- local results = http.pipeline_go('nmap.org', 80, all)
37--</code>
38--
39-- At this point, <code>results</code> is an array with three elements. Each element
40-- is a table containing the HTTP result, as discussed above.
41--
42-- One more interface provided by the HTTP library helps scripts determine whether or not
43-- a page exists. The <code>identify_404</code> function will try several URLs on the
44-- server to determine what the server's 404 pages look like. It will attempt to identify
45-- customized 404 pages that may not return the actual status code 404. If successful,
46-- the function <code>page_exists</code> can then be used to determine whether or not
47-- a page existed.
48--
49-- Some other miscellaneous functions that can come in handy are <code>response_contains</code>,
50-- <code>can_use_head</code>, and <code>save_path</code>. See the appropriate documentation
51-- for them.
52--
53-- The response to each function is typically a table with the following keys:
54-- * <code>status-line</code>: The HTTP status line; for example, "HTTP/1.1 200 OK" (note: this is followed by a newline). In case of an error, a description will be provided in this line.
55-- * <code>status</code>: The HTTP status value; for example, "200". If an error occurs during a request, then this value is going to be nil.
56-- * <code>header</code>: A table of header values, where the keys are lowercase and the values are exactly what the server sent
57-- * <code>rawheader</code>: A list of header values as "name: value" strings, in the exact format and order that the server sent them
58-- * <code>cookies</code>: A list of cookies that the server is sending. Each cookie is a table containing the keys <code>name</code>, <code>value</code>, and <code>path</code>. This table can be sent to the server in subsequent responses in the <code>options</code> table to any function (see below).
59-- * <code>body</code>: The body of the response
60-- * <code>location</code>: a list of the locations of redirects that were followed.
61--
62-- Many of the functions optionally allow an 'options' table. This table can alter the HTTP headers
63-- or other values like the timeout. The following are valid values in 'options' (note: not all
64-- options will necessarily affect every function):
65-- * <code>timeout</code>: A timeout used for socket operations.
66-- * <code>header</code>: A table containing additional headers to be used for the request. For example, <code>options['header']['Content-Type'] = 'text/xml'</code>
67-- * <code>content</code>: The content of the message (content-length will be added -- set header['Content-Length'] to override). This can be either a string, which will be directly added as the body of the message, or a table, which will have each key=value pair added (like a normal POST request).
68-- * <code>cookies</code>: A list of cookies as either a string, which will be directly sent, or a table. If it's a table, the following fields are recognized: <code>name</code>, <code>value</code>, <code>path</code>, <code>expires</code>. Only <code>name</code> and <code>value</code> fields are required.
69-- * <code>auth</code>: A table containing the keys <code>username</code> and <code>password</code>, which will be used for HTTP Basic authentication.
70-- If a server requires HTTP Digest authentication, then there must also be a key <code>digest</code>, with value <code>true</code>.
71-- If a server requires NTLM authentication, then there must also be a key <code>ntlm</code>, with value <code>true</code>.
72-- * <code>bypass_cache</code>: Do not perform a lookup in the local HTTP cache.
73-- * <code>no_cache</code>: Do not save the result of this request to the local HTTP cache.
74-- * <code>no_cache_body</code>: Do not save the body of the response to the local HTTP cache.
75-- * <code>any_af</code>: Allow connecting to any address family, inet or inet6. By default, these functions will only use the same AF as nmap.address_family to resolve names.
76-- * <code>redirect_ok</code>: Closure that overrides the default redirect_ok used to validate whether to follow HTTP redirects or not. False, if no HTTP redirects should be followed. Alternatively, a number may be passed to change the number of redirects to follow.
77-- The following example shows how to write a custom closure that follows 5 consecutive redirects, without the safety checks in the default redirect_ok:
78-- <code>
79-- redirect_ok = function(host,port)
80-- local c = 5
81-- return function(url)
82-- if ( c==0 ) then return false end
83-- c = c - 1
84-- return true
85-- end
86-- end
87-- </code>
88--
89-- @args http.max-cache-size The maximum memory size (in bytes) of the cache.
90--
91-- @args http.useragent The value of the User-Agent header field sent with
92-- requests. By default it is
93-- <code>"Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)"</code>.
94-- A value of the empty string disables sending the User-Agent header field.
95--
96-- @args http.pipeline If set, it represents the number of HTTP requests that'll be
97-- sent on one connection. This can be set low to make debugging easier, or it
98-- can be set high to test how a server reacts (its chosen max is ignored).
99-- @args http.max-pipeline If set, it represents the number of outstanding HTTP requests
100-- that should be pipelined. Defaults to <code>http.pipeline</code> (if set), or to what
101-- <code>getPipelineMax</code> function returns.
102--
103-- TODO
104-- Implement cache system for http pipelines
105--
106
107
108local base64 = require "base64"
109local bin = require "bin"
110local bit = require "bit"
111local comm = require "comm"
112local coroutine = require "coroutine"
113local nmap = require "nmap"
114local os = require "os"
115local sasl = require "sasl"
116local slaxml = require "slaxml"
117local stdnse = require "stdnse"
118local string = require "string"
119local table = require "table"
120local url = require "url"
121local smbauth = require "smbauth"
122local unicode = require "unicode"
123
124_ENV = stdnse.module("http", stdnse.seeall)
125
126---Use ssl if we have it
127local have_ssl, openssl = pcall(require,'openssl')
128
129USER_AGENT = stdnse.get_script_args('http.useragent') or "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)"
130local MAX_REDIRECT_COUNT = 5
131
132-- Recursively copy a table.
133-- Only recurs when a value is a table, other values are copied by assignment.
134local function tcopy (t)
135 local tc = {};
136 for k,v in pairs(t) do
137 if type(v) == "table" then
138 tc[k] = tcopy(v);
139 else
140 tc[k] = v;
141 end
142 end
143 return tc;
144end
145
146--- Recursively copy into a table any elements from another table whose key it
147-- doesn't have.
148local function table_augment(to, from)
149 for k, v in pairs(from) do
150 if type( to[k] ) == 'table' then
151 table_augment(to[k], from[k])
152 else
153 to[k] = from[k]
154 end
155 end
156end
157
158--- Get a value suitable for the Host header field.
159-- See RFC 2616 sections 14.23 and 5.2.
160local function get_host_field(host, port)
161 return stdnse.get_hostname(host)
162end
163
164-- Skip *( SP | HT ) starting at offset. See RFC 2616, section 2.2.
165-- @return the first index following the spaces.
166-- @return the spaces skipped over.
167local function skip_space(s, offset)
168 local _, i, space = s:find("^([ \t]*)", offset)
169 return i + 1, space
170end
171
172-- Get a token starting at offset. See RFC 2616, section 2.2.
173-- @return the first index following the token, or nil if no token was found.
174-- @return the token.
175local function get_token(s, offset)
176 -- All characters except CTL and separators.
177 local _, i, token = s:find("^([^()<>@,;:\\\"/%[%]?={} \0\001-\031\127]+)", offset)
178 if i then
179 return i + 1, token
180 else
181 return nil
182 end
183end
184
185-- Get a quoted-string starting at offset. See RFC 2616, section 2.2. crlf is
186-- used as the definition for CRLF in the case of LWS within the string.
187-- @return the first index following the quoted-string, or nil if no
188-- quoted-string was found.
189-- @return the contents of the quoted-string, without quotes or backslash
190-- escapes.
191local function get_quoted_string(s, offset, crlf)
192 local result = {}
193 local i = offset
194 assert(s:sub(i, i) == "\"")
195 i = i + 1
196 while i <= s:len() do
197 local c = s:sub(i, i)
198 if c == "\"" then
199 -- Found the closing quote, done.
200 return i + 1, table.concat(result)
201 elseif c == "\\" then
202 -- This is a quoted-pair ("\" CHAR).
203 i = i + 1
204 c = s:sub(i, i)
205 if c == "" then
206 -- No character following.
207 error("\\ escape at end of input while parsing quoted-string.")
208 end
209 -- Only CHAR may follow a backslash.
210 if c:byte(1) > 127 then
211 error(string.format("Unexpected character with value > 127 (0x%02X) in quoted-string.", c:byte(1)))
212 end
213 else
214 -- This is qdtext, which is TEXT except for '"'.
215 -- TEXT is "any OCTET except CTLs, but including LWS," however "a CRLF is
216 -- allowed in the definition of TEXT only as part of a header field
217 -- continuation." So there are really two definitions of quoted-string,
218 -- depending on whether it's in a header field or not. This function does
219 -- not allow CRLF.
220 c = s:sub(i, i)
221 if c ~= "\t" and c:match("^[\0\001-\031\127]$") then
222 error(string.format("Unexpected control character in quoted-string: 0x%02X.", c:byte(1)))
223 end
224 end
225 result[#result + 1] = c
226 i = i + 1
227 end
228 return nil
229end
230
231-- Returns the index just past the end of LWS.
232local function skip_lws(s, pos)
233 local _, e
234
235 while true do
236 while string.match(s, "^[ \t]", pos) do
237 pos = pos + 1
238 end
239 _, e = string.find(s, "^\r?\n[ \t]", pos)
240 if not e then
241 return pos
242 end
243 pos = e + 1
244 end
245end
246
247
248---Validate an 'options' table, which is passed to a number of the HTTP functions. It is
249-- often difficult to track down a mistake in the options table, and requires fiddling
250-- with the http.lua source, but this should make that a lot easier.
251local function validate_options(options)
252 local bad = false
253
254 if(options == nil) then
255 return true
256 end
257
258 for key, value in pairs(options) do
259 if(key == 'timeout') then
260 if(type(tonumber(value)) ~= 'number') then
261 stdnse.debug1('http: options.timeout contains a non-numeric value')
262 bad = true
263 end
264 elseif(key == 'header') then
265 if(type(value) ~= 'table') then
266 stdnse.debug1("http: options.header should be a table")
267 bad = true
268 end
269 elseif(key == 'content') then
270 if(type(value) ~= 'string' and type(value) ~= 'table') then
271 stdnse.debug1("http: options.content should be a string or a table")
272 bad = true
273 end
274 elseif(key == 'cookies') then
275 if(type(value) == 'table') then
276 for _, cookie in ipairs(value) do
277 for cookie_key, cookie_value in pairs(cookie) do
278 if(cookie_key == 'name') then
279 if(type(cookie_value) ~= 'string') then
280 stdnse.debug1("http: options.cookies[i].name should be a string")
281 bad = true
282 end
283 elseif(cookie_key == 'value') then
284 if(type(cookie_value) ~= 'string') then
285 stdnse.debug1("http: options.cookies[i].value should be a string")
286 bad = true
287 end
288 elseif(cookie_key == 'path') then
289 if(type(cookie_value) ~= 'string') then
290 stdnse.debug1("http: options.cookies[i].path should be a string")
291 bad = true
292 end
293 elseif(cookie_key == 'expires') then
294 if(type(cookie_value) ~= 'string') then
295 stdnse.debug1("http: options.cookies[i].expires should be a string")
296 bad = true
297 end
298 elseif(cookie_key == 'max-age') then
299 if(type(cookie_value) ~= 'string') then
300 stdnse.debug1("http: options.cookies[i].max-age should be a string")
301 bad = true
302 end
303 elseif not (cookie_key == 'httponly' or cookie_key == 'secure') then
304 stdnse.debug1("http: Unknown field in cookie table: %s", cookie_key)
305 bad = true
306 end
307 end
308 end
309 elseif(type(value) ~= 'string') then
310 stdnse.debug1("http: options.cookies should be a table or a string")
311 bad = true
312 end
313 elseif(key == 'auth') then
314 if(type(value) == 'table') then
315 if(value['username'] == nil or value['password'] == nil) then
316 stdnse.debug1("http: options.auth should contain both a 'username' and a 'password' key")
317 bad = true
318 end
319 else
320 stdnse.debug1("http: options.auth should be a table")
321 bad = true
322 end
323 elseif (key == 'digestauth') then
324 if(type(value) == 'table') then
325 local req_keys = {"username","realm","nonce","digest-uri","response"}
326 for _,k in ipairs(req_keys) do
327 if not value[k] then
328 stdnse.debug1("http: options.digestauth missing key: %s",k)
329 bad = true
330 break
331 end
332 end
333 else
334 bad = true
335 stdnse.debug1("http: options.digestauth should be a table")
336 end
337 elseif (key == 'ntlmauth') then
338 stdnse.debug1("Proceeding with ntlm message")
339 elseif(key == 'bypass_cache' or key == 'no_cache' or key == 'no_cache_body' or key == 'any_af') then
340 if(type(value) ~= 'boolean') then
341 stdnse.debug1("http: options.bypass_cache, options.no_cache, and options.no_cache_body must be boolean values")
342 bad = true
343 end
344 elseif(key == 'redirect_ok') then
345 if(type(value)~= 'function' and type(value)~='boolean' and type(value) ~= 'number') then
346 stdnse.debug1("http: options.redirect_ok must be a function or boolean or number")
347 bad = true
348 end
349 else
350 stdnse.debug1("http: Unknown key in the options table: %s", key)
351 end
352 end
353
354 return not(bad)
355end
356
357
358-- The following recv functions, and the function <code>next_response</code>
359-- follow a common pattern. They each take a <code>partial</code> argument
360-- whose value is data that has been read from the socket but not yet used in
361-- parsing, and they return as their second return value a new value for
362-- <code>partial</code>. The idea is that, for example, in reading from the
363-- socket to get the Status-Line, you will probably read too much and read part
364-- of the header. That part (the "partial") has to be retained when you go to
365-- parse the header. The common use pattern is this:
366-- <code>
367-- local partial
368-- status_line, partial = recv_line(socket, partial)
369-- ...
370-- header, partial = recv_header(socket, partial)
371-- ...
372-- </code>
373-- On error, the functions return <code>nil</code> and the second return value
374-- is an error message.
375
376-- Receive a single line (up to <code>\n</code>).
377local function recv_line(s, partial)
378 local _, e
379 local status, data
380 local pos
381
382 partial = partial or ""
383
384 pos = 1
385 while true do
386 _, e = string.find(partial, "\n", pos, true)
387 if e then
388 break
389 end
390 status, data = s:receive()
391 if not status then
392 return status, data
393 end
394 pos = #partial
395 partial = partial .. data
396 end
397
398 return string.sub(partial, 1, e), string.sub(partial, e + 1)
399end
400
401local function line_is_empty(line)
402 return line == "\r\n" or line == "\n"
403end
404
405-- Receive up to and including the first blank line, but return everything up
406-- to and not including the final blank line.
407local function recv_header(s, partial)
408 local lines = {}
409
410 partial = partial or ""
411
412 while true do
413 local line
414 line, partial = recv_line(s, partial)
415 if not line then
416 return line, partial
417 end
418 if line_is_empty(line) then
419 break
420 end
421 lines[#lines + 1] = line
422 end
423
424 return table.concat(lines), partial
425end
426
427-- Receive until the connection is closed.
428local function recv_all(s, partial)
429 local parts
430
431 partial = partial or ""
432
433 parts = {partial}
434 while true do
435 local status, part = s:receive()
436 if not status then
437 break
438 else
439 parts[#parts + 1] = part
440 end
441 end
442
443 return table.concat(parts), ""
444end
445
446-- Receive exactly <code>length</code> bytes. Returns <code>nil</code> if that
447-- many aren't available.
448local function recv_length(s, length, partial)
449 local parts, last
450
451 partial = partial or ""
452
453 parts = {}
454 last = partial
455 length = length - #last
456 while length > 0 do
457 local status
458
459 parts[#parts + 1] = last
460 status, last = s:receive()
461
462 return nil, table.concat( parts)
463
464 length = length - #last
465 end
466
467 -- At this point length is 0 or negative, and indicates the degree to which
468 -- the last read "overshot" the desired length.
469
470 if length == 0 then
471 return table.concat(parts) .. last, ""
472 else
473 return table.concat(parts) .. string.sub(last, 1, length - 1), string.sub(last, length)
474 end
475end
476
477-- Receive until the end of a chunked message body, and return the dechunked
478-- body.
479local function recv_chunked(s, partial)
480 local chunks, chunk
481 local chunk_size
482 local pos
483
484 chunks = {}
485 repeat
486 local line, hex, _, i
487
488 line, partial = recv_line(s, partial)
489 if not line then
490 return nil, partial
491 end
492
493 pos = 1
494 pos = skip_space(line, pos)
495
496 -- Get the chunk-size.
497 _, i, hex = string.find(line, "^([%x]+)", pos)
498 if not i then
499 return nil, string.format("Chunked encoding didn't find hex; got %q.", string.sub(line, pos, pos + 10))
500 end
501 pos = i + 1
502
503 chunk_size = tonumber(hex, 16)
504 if not chunk_size or chunk_size < 0 then
505 return nil, string.format("Chunk size %s is not a positive integer.", hex)
506 end
507
508 -- Ignore chunk-extensions that may follow here.
509 -- RFC 2616, section 2.1 ("Implied *LWS") seems to allow *LWS between the
510 -- parts of a chunk-extension, but that is ambiguous. Consider this case:
511 -- "1234;a\r\n =1\r\n...". It could be an extension with a chunk-ext-name
512 -- of "a" (and no value), and a chunk-data beginning with " =", or it could
513 -- be a chunk-ext-name of "a" with a value of "1", and a chunk-data
514 -- starting with "...". We don't allow *LWS here, only ( SP | HT ), so the
515 -- first interpretation will prevail.
516
517 chunk, partial = recv_length(s, chunk_size, partial)
518 if not chunk then
519 return nil, table.concat(chunks) .. (partial or "")
520 end
521 chunks[#chunks + 1] = chunk
522
523 line, partial = recv_line(s, partial)
524 if not line then
525 -- this warning message was initially an error but was adapted
526 -- to support broken servers, such as the Citrix XML Service
527 stdnse.debug2("Didn't find CRLF after chunk-data.")
528 elseif not string.match(line, "^\r?\n") then
529 return nil, string.format("Didn't find CRLF after chunk-data; got %q.", line)
530 end
531 until chunk_size == 0
532
533 return table.concat(chunks), partial
534end
535
536-- Receive a message body, assuming that the header has already been read by
537-- <code>recv_header</code>. The handling is sensitive to the request method
538-- and the status code of the response.
539local function recv_body(s, response, method, partial)
540 local connection_close, connection_keepalive
541 local version_major, version_minor
542 local transfer_encoding
543 local content_length
544 local err
545
546 partial = partial or ""
547
548 -- First check for Connection: close and Connection: keep-alive. This is
549 -- necessary to handle some servers that don't follow the protocol.
550 connection_close = false
551 connection_keepalive = false
552 if response.header.connection then
553 local offset, token
554 offset = 0
555 while true do
556 offset, token = get_token(response.header.connection, offset + 1)
557 if not offset then
558 break
559 end
560 if string.lower(token) == "close" then
561 connection_close = true
562 elseif string.lower(token) == "keep-alive" then
563 connection_keepalive = true
564 end
565 end
566 end
567
568 -- The HTTP version may also affect our decisions.
569 version_major, version_minor = string.match(response["status-line"], "^HTTP/(%d+)%.(%d+)")
570
571 -- See RFC 2616, section 4.4 "Message Length".
572
573 -- 1. Any response message which "MUST NOT" include a message-body (such as
574 -- the 1xx, 204, and 304 responses and any response to a HEAD request) is
575 -- always terminated by the first empty line after the header fields...
576 --
577 -- Despite the above, some servers return a body with response to a HEAD
578 -- request. So if an HTTP/1.0 server returns a response without Connection:
579 -- keep-alive, or any server returns a response with Connection: close, read
580 -- whatever's left on the socket (should be zero bytes).
581 if string.upper(method) == "HEAD"
582 or (response.status >= 100 and response.status <= 199)
583 or response.status == 204 or response.status == 304 then
584 if connection_close or (version_major == "1" and version_minor == "0" and not connection_keepalive) then
585 return recv_all(s, partial)
586 else
587 return "", partial
588 end
589 end
590
591 -- 2. If a Transfer-Encoding header field (section 14.41) is present and has
592 -- any value other than "identity", then the transfer-length is defined by
593 -- use of the "chunked" transfer-coding (section 3.6), unless the message
594 -- is terminated by closing the connection.
595 if response.header["transfer-encoding"]
596 and response.header["transfer-encoding"] ~= "identity" then
597 return recv_chunked(s, partial)
598 end
599 -- The Citrix XML Service sends a wrong "Transfer-Coding" instead of
600 -- "Transfer-Encoding".
601 if response.header["transfer-coding"]
602 and response.header["transfer-coding"] ~= "identity" then
603 return recv_chunked(s, partial)
604 end
605
606 -- 3. If a Content-Length header field (section 14.13) is present, its decimal
607 -- value in OCTETs represents both the entity-length and the
608 -- transfer-length. The Content-Length header field MUST NOT be sent if
609 -- these two lengths are different (i.e., if a Transfer-Encoding header
610 -- field is present). If a message is received with both a
611 -- Transfer-Encoding header field and a Content-Length header field, the
612 -- latter MUST be ignored.
613 if response.header["content-length"] and not response.header["transfer-encoding"] then
614 content_length = tonumber(response.header["content-length"])
615 if not content_length then
616 return nil, string.format("Content-Length %q is non-numeric", response.header["content-length"])
617 end
618 return recv_length(s, content_length, partial)
619 end
620
621 -- 4. If the message uses the media type "multipart/byteranges", and the
622 -- transfer-length is not otherwise specified, then this self-delimiting
623 -- media type defines the transfer-length. [sic]
624
625 -- Case 4 is unhandled.
626
627 -- 5. By the server closing the connection.
628 return recv_all(s, partial)
629end
630
631-- Sets response["status-line"] and response.status.
632local function parse_status_line(status_line, response)
633 local version, status, reason_phrase
634
635 response["status-line"] = status_line
636 version, status, reason_phrase = string.match(status_line,
637 "^HTTP/(%d%.%d) *(%d+) *(.*)\r?\n$")
638 if not version then
639 return nil, string.format("Error parsing status-line %q.", status_line)
640 end
641 -- We don't have a use for the version; ignore it.
642 response.status = tonumber(status)
643 if not response.status then
644 return nil, string.format("Status code is not numeric: %s", status)
645 end
646
647 return true
648end
649
650local parse_set_cookie -- defined farther down
651
652-- Sets response.header and response.rawheader.
653local function parse_header(header, response)
654 local pos
655 local name, words
656 local s, e
657
658 response.header = {}
659 response.rawheader = stdnse.strsplit("\r?\n", header)
660 pos = 1
661 while pos <= #header do
662 -- Get the field name.
663 e, name = get_token(header, pos)
664 -- Do not bail out if the header is malformed. Consume the header line
665 -- anyway, getting to the next header, but do not create a new entry in
666 -- the "header" table.
667 if e then
668 if header:sub(e, e) ~= ":" then
669 name = nil
670 end
671 pos = e + 1
672 end
673
674 -- Skip initial space.
675 pos = skip_lws(header, pos)
676 -- Get non-space words separated by LWS, then join them with a single space.
677 words = {}
678 while pos <= #header and not string.match(header, "^\r?\n", pos) do
679 s = pos
680 while not string.match(header, "^[ \t]", pos) and
681 not string.match(header, "^\r?\n", pos) do
682 pos = pos + 1
683 end
684 words[#words + 1] = string.sub(header, s, pos - 1)
685 pos = skip_lws(header, pos)
686 end
687
688 if name then
689 -- Set it in our table.
690 name = string.lower(name)
691 local value = table.concat(words, " ")
692 if response.header[name] then
693 -- TODO: delay concatenation until return to avoid resource exhaustion
694 response.header[name] = response.header[name] .. ", " .. value
695 else
696 response.header[name] = value
697 end
698
699 -- Update the cookie table if this is a Set-Cookie header
700 if name == "set-cookie" then
701 local cookie, err = parse_set_cookie(value)
702 if cookie then
703 response.cookies[#response.cookies + 1] = cookie
704 else
705 -- Ignore any cookie parsing error
706 end
707 end
708 end
709
710 -- Next field, or end of string. (If not it's an error.)
711 s, e = string.find(header, "^\r?\n", pos)
712 if not e then
713 return nil, string.format("Header field named %q didn't end with CRLF", name)
714 end
715 pos = e + 1
716 end
717
718 return true
719end
720
721-- Parse the contents of a Set-Cookie header field.
722-- The result is a table of the form
723--
724-- { name = "NAME", value = "VALUE", Comment = "...", Domain = "...", ... }
725--
726-- Every key except "name" and "value" is optional.
727--
728-- This function attempts to support the cookie syntax defined in RFC 2109
729-- along with the backwards-compatibility suggestions from its section 10,
730-- "HISTORICAL". Values need not be quoted, but if they start with a quote they
731-- will be interpreted as a quoted string.
732parse_set_cookie = function (s)
733 local name, value
734 local _
735
736 local cookie = {}
737
738 -- Get the NAME=VALUE part.
739 local pos = skip_space(s, 1)
740 pos, cookie.name = get_token(s, pos)
741 if not cookie.name then
742 return nil, "Can't get cookie name."
743 end
744 pos = skip_space(s, pos)
745 if s:sub(pos, pos) ~= "=" then
746 return nil, string.format("Expected '=' after cookie name \"%s\".", cookie.name)
747 end
748 pos = pos + 1
749 pos = skip_space(s, pos)
750 if s:sub(pos, pos) == "\"" then
751 pos, cookie.value = get_quoted_string(s, pos)
752 else
753 _, pos, cookie.value = s:find("([^; \t]*)", pos)
754 pos = pos + 1
755 end
756 if not cookie.value then
757 return nil, string.format("Can't get value of cookie named \"%s\".", cookie.name)
758 end
759 pos = skip_space(s, pos)
760
761 -- Loop over the attributes.
762 while s:sub(pos, pos) == ";" do
763 pos = pos + 1
764 pos = skip_space(s, pos)
765 pos, name = get_token(s, pos)
766 if not name then
767 return nil, string.format("Can't get attribute name of cookie \"%s\".", cookie.name)
768 end
769 pos = skip_space(s, pos)
770 if s:sub(pos, pos) == "=" then
771 pos = pos + 1
772 pos = skip_space(s, pos)
773 if s:sub(pos, pos) == "\"" then
774 pos, value = get_quoted_string(s, pos)
775 else
776 _, pos, value = s:find("([^;]*)", pos)
777 value = value:match("(.-)[ \t]*$")
778 pos = pos + 1
779 end
780 if not value then
781 return nil, string.format("Can't get value of cookie attribute \"%s\".", name)
782 end
783 else
784 value = true
785 end
786 cookie[name:lower()] = value
787 pos = skip_space(s, pos)
788 end
789
790 return cookie
791end
792
793-- Read one response from the socket <code>s</code> and return it after
794-- parsing.
795local function next_response(s, method, partial)
796 local response
797 local status_line, header, body
798 local status, err
799
800 partial = partial or ""
801 response = {
802 status=nil,
803 ["status-line"]=nil,
804 header={},
805 rawheader={},
806 cookies={},
807 body=""
808 }
809
810 status_line, partial = recv_line(s, partial)
811 if not status_line then
812 return nil, partial
813 end
814 status, err = parse_status_line(status_line, response)
815 if not status then
816 return nil, err
817 end
818
819 header, partial = recv_header(s, partial)
820 if not header then
821 return nil, partial
822 end
823 status, err = parse_header(header, response)
824 if not status then
825 return nil, err
826 end
827
828 body, partial = recv_body(s, response, method, partial)
829 if not body then
830 return nil, partial
831 end
832 response.body = body
833
834 return response, partial
835end
836
837--- Tries to extract the max number of requests that should be made on
838-- a keep-alive connection based on "Keep-Alive: timeout=xx,max=yy" response
839-- header.
840--
841-- If the value is not available, an arbitrary value is used. If the connection
842-- is not explicitly closed by the server, this same value is attempted.
843--
844-- @param response The http response - Might be a table or a raw response
845-- @return The max number of requests on a keep-alive connection
846local function getPipelineMax(response)
847 -- Allow users to override this with a script-arg
848 local pipeline = stdnse.get_script_args({'http.pipeline', 'pipeline'})
849
850 if(pipeline) then
851 return tonumber(pipeline)
852 end
853
854 if response then
855 if response.header and response.header.connection ~= "close" then
856 if response.header["keep-alive"] then
857 local max = string.match( response.header["keep-alive"], "max=(%d*)")
858 if(max == nil) then
859 return 40
860 end
861 return tonumber(max)
862 else
863 return 40
864 end
865 end
866 end
867 return 1
868end
869
870--- Builds a string to be added to the request mod_options table
871--
872-- @param cookies A cookie jar just like the table returned parse_set_cookie.
873-- @param path If the argument exists, only cookies with this path are included to the request
874-- @return A string to be added to the mod_options table
875local function buildCookies(cookies, path)
876 if type(cookies) == 'string' then return cookies end
877 local cookie = {}
878 for _, ck in ipairs(cookies or {}) do
879 local ckpath = ck["path"]
880 if not path or not ckpath
881 or ckpath == path
882 or ckpath:sub(-1) == "/" and ckpath == path:sub(1, ckpath:len())
883 or ckpath .. "/" == path:sub(1, ckpath:len()+1)
884 then
885 cookie[#cookie+1] = ck["name"] .. "=" .. ck["value"]
886 end
887 end
888 return table.concat(cookie, "; ")
889end
890
891-- HTTP cache.
892-- Cache of GET and HEAD requests. Uses <"host:port:path", record>.
893-- record is in the format:
894-- result: The result from http.get or http.head
895-- last_used: The time the record was last accessed or made.
896-- get: Was the result received from a request to get or recently wiped?
897-- size: The size of the record, equal to #record.result.body.
898local cache = {size = 0};
899
900local function check_size (cache)
901 local max_size = tonumber(stdnse.get_script_args({'http.max-cache-size', 'http-max-cache-size'}) or 1e6);
902
903 local size = cache.size;
904
905 if size > max_size then
906 stdnse.debug1(
907 "Current http cache size (%d bytes) exceeds max size of %d",
908 size, max_size);
909 table.sort(cache, function(r1, r2)
910 return (r1.last_used or 0) < (r2.last_used or 0);
911 end);
912
913 for i, record in ipairs(cache) do
914 if size <= max_size then break end
915 local result = record.result;
916 if type(result.body) == "string" then
917 size = size - record.size;
918 record.size, record.get, result.body = 0, false, "";
919 end
920 end
921 cache.size = size;
922 end
923 stdnse.debug2("Final http cache size (%d bytes) of max size of %d",
924 size, max_size);
925 return size;
926end
927
928-- Unique value to signal value is being retrieved.
929-- Also holds <mutex, thread> pairs, working thread is value
930local WORKING = setmetatable({}, {__mode = "v"});
931
932local function lookup_cache (method, host, port, path, options)
933 if(not(validate_options(options))) then
934 return nil
935 end
936
937 options = options or {};
938 local bypass_cache = options.bypass_cache; -- do not lookup
939 local no_cache = options.no_cache; -- do not save result
940 local no_cache_body = options.no_cache_body; -- do not save body
941
942 if type(port) == "table" then port = port.number end
943
944 local key = stdnse.get_hostname(host)..":"..port..":"..path;
945 local mutex = nmap.mutex(tostring(lookup_cache)..key);
946
947 local state = {
948 mutex = mutex,
949 key = key,
950 method = method,
951 bypass_cache = bypass_cache,
952 no_cache = no_cache,
953 no_cache_body = no_cache_body,
954 };
955
956 while true do
957 mutex "lock";
958 local record = cache[key];
959 if bypass_cache or record == nil or method ~= record.method then
960 WORKING[mutex] = coroutine.running();
961 cache[key], state.old_record = WORKING, record;
962 return nil, state;
963 elseif record == WORKING then
964 local working = WORKING[mutex];
965 if working == nil or coroutine.status(working) == "dead" then
966 -- thread died before insert_cache could be called
967 cache[key] = nil; -- reset
968 end
969 mutex "done";
970 else
971 mutex "done";
972 record.last_used = os.time();
973 return tcopy(record.result), state;
974 end
975 end
976end
977
978local function response_is_cacheable(response)
979 -- if response.status is nil, then an error must have occurred during the request
980 -- and we probably don't want to cache the response
981 if not response.status then
982 return false
983 end
984
985 -- 206 Partial Content. RFC 2616, 1.34: "...a cache that does not support the
986 -- Range and Content-Range headers MUST NOT cache 206 (Partial Content)
987 -- responses."
988 if response.status == 206 then
989 return false
990 end
991
992 -- RFC 2616, 13.4. "A response received with any [status code other than 200,
993 -- 203, 206, 300, 301 or 410] (e.g. status codes 302 and 307) MUST NOT be
994 -- returned in a reply to a subsequent request unless there are cache-control
995 -- directives or another header(s) that explicitly allow it."
996 -- We violate the standard here and allow these other codes to be cached,
997 -- with the exceptions listed below.
998
999 -- 401 Unauthorized. Caching this would prevent us from retrieving it later
1000 -- with the correct credentials.
1001 if response.status == 401 then
1002 return false
1003 end
1004
1005 return true
1006end
1007
1008local function insert_cache (state, response)
1009 local key = assert(state.key);
1010 local mutex = assert(state.mutex);
1011
1012 if response == nil or state.no_cache or not response_is_cacheable(response) then
1013 cache[key] = state.old_record;
1014 else
1015 local record = {
1016 result = tcopy(response),
1017 last_used = os.time(),
1018 method = state.method,
1019 size = type(response.body) == "string" and #response.body or 0,
1020 };
1021 response = record.result; -- only modify copy
1022 cache[key], cache[#cache+1] = record, record;
1023 if state.no_cache_body then
1024 response.body = "";
1025 end
1026 if type(response.body) == "string" then
1027 cache.size = cache.size + #response.body;
1028 check_size(cache);
1029 end
1030 end
1031 mutex "done";
1032end
1033
1034-- Return true if the given method requires a body in the request. In case no
1035-- body was supplied we must send "Content-Length: 0".
1036local function request_method_needs_content_length(method)
1037 return method == "POST"
1038end
1039
1040-- For each of the following request functions, <code>host</code> may either be
1041-- a string or a table, and <code>port</code> may either be a number or a
1042-- table.
1043--
1044-- The format of the return value is a table with the following structure:
1045-- {status = 200, status-line = "HTTP/1.1 200 OK", header = {}, rawheader = {}, body ="<html>...</html>"}
1046-- The header table has an entry for each received header with the header name
1047-- being the key. The table also has an entry named "status" which contains the
1048-- http status code of the request.
1049-- In case of an error, the status is nil and status-line describes the problem.
1050
1051local function http_error(status_line, partial)
1052 return {
1053 status = nil,
1054 ["status-line"] = status_line,
1055 header = {},
1056 rawheader = {},
1057 body = nil,
1058 partial = partial
1059 }
1060end
1061
1062--- Build an HTTP request from parameters and return it as a string.
1063--
1064-- @param host The host this request is intended for.
1065-- @param port The port this request is intended for.
1066-- @param method The method to use.
1067-- @param path The path for the request.
1068-- @param options A table of options, which may include the keys:
1069-- * <code>header</code>: A table containing additional headers to be used for the request.
1070-- * <code>content</code>: The content of the message (content-length will be added -- set header['Content-Length'] to override)
1071-- * <code>cookies</code>: A table of cookies in the form returned by <code>parse_set_cookie</code>.
1072-- * <code>auth</code>: A table containing the keys <code>username</code> and <code>password</code>.
1073-- @return A request string.
1074-- @see generic_request
1075local function build_request(host, port, method, path, options)
1076 if(not(validate_options(options))) then
1077 return nil
1078 end
1079 options = options or {}
1080
1081 -- Private copy of the options table, used to add default header fields.
1082 local mod_options = {
1083 header = {
1084 Connection = "close",
1085 Host = get_host_field(host, port),
1086 ["User-Agent"] = USER_AGENT
1087 }
1088 }
1089
1090 if options.cookies then
1091 local cookies = buildCookies(options.cookies, path)
1092 if #cookies > 0 then
1093 mod_options.header["Cookie"] = cookies
1094 end
1095 end
1096
1097 if options.auth and not (options.auth.digest or options.auth.ntlm) then
1098 local username = options.auth.username
1099 local password = options.auth.password
1100 local credentials = "Basic " .. base64.enc(username .. ":" .. password)
1101 mod_options.header["Authorization"] = credentials
1102 end
1103
1104 if options.digestauth then
1105 local order = {"username", "realm", "nonce", "digest-uri", "algorithm", "response", "qop", "nc", "cnonce"}
1106 local no_quote = {algorithm=true, qop=true, nc=true}
1107 local creds = {}
1108 for _,k in ipairs(order) do
1109 local v = options.digestauth[k]
1110 if v then
1111 if no_quote[k] then
1112 table.insert(creds, ("%s=%s"):format(k,v))
1113 else
1114 if k == "digest-uri" then
1115 table.insert(creds, ('%s="%s"'):format("uri",v))
1116 else
1117 table.insert(creds, ('%s="%s"'):format(k,v))
1118 end
1119 end
1120 end
1121 end
1122 local credentials = "Digest "..table.concat(creds, ", ")
1123 mod_options.header["Authorization"] = credentials
1124 end
1125
1126 if options.ntlmauth then
1127 mod_options.header["Authorization"] = "NTLM " .. base64.enc(options.ntlmauth)
1128 end
1129
1130
1131 local body
1132 -- Build a form submission from a table, like "k1=v1&k2=v2".
1133 if type(options.content) == "table" then
1134 local parts = {}
1135 local k, v
1136 for k, v in pairs(options.content) do
1137 parts[#parts + 1] = url.escape(k) .. "=" .. url.escape(v)
1138 end
1139 body = table.concat(parts, "&")
1140 mod_options.header["Content-Type"] = "application/x-www-form-urlencoded"
1141 elseif options.content then
1142 body = options.content
1143 elseif request_method_needs_content_length(method) then
1144 body = ""
1145 end
1146 if body then
1147 mod_options.header["Content-Length"] = #body
1148 end
1149
1150 -- Add any other header fields into the local copy.
1151 table_augment(mod_options, options)
1152 -- We concat this string manually to allow null bytes in requests
1153 local request_line = method.." "..path.." HTTP/1.1"
1154 local header = {}
1155 for name, value in pairs(mod_options.header) do
1156 -- we concat this string manually to allow null bytes in requests
1157 header[#header + 1] = name..": "..value
1158 end
1159
1160 return request_line .. "\r\n" .. stdnse.strjoin("\r\n", header) .. "\r\n\r\n" .. (body or "")
1161end
1162
1163--- Send a string to a host and port and return the HTTP result. This function
1164-- is like <code>generic_request</code>, to be used when you have a ready-made
1165-- request, not a collection of request parameters.
1166--
1167-- @param host The host to connect to.
1168-- @param port The port to connect to.
1169-- @param options A table of other parameters. It may have any of these fields:
1170-- * <code>timeout</code>: A timeout used for socket operations.
1171-- * <code>header</code>: A table containing additional headers to be used for the request.
1172-- * <code>content</code>: The content of the message (content-length will be added -- set header['Content-Length'] to override)
1173-- * <code>cookies</code>: A table of cookies in the form returned by <code>parse_set_cookie</code>.
1174-- * <code>auth</code>: A table containing the keys <code>username</code> and <code>password</code>.
1175-- @return A response table, see module documentation for description.
1176-- @see generic_request
1177local function request(host, port, data, options)
1178 if(not(validate_options(options))) then
1179 return http_error("Options failed to validate.")
1180 end
1181 local method
1182 local header
1183 local response
1184 stdnse.debug1("+++++++++++++++++++++++++++++=lololol=+++++++++++++++++++++++++++++")
1185
1186 options = options or {}
1187
1188 if type(port) == 'table' then
1189 if port.protocol and port.protocol ~= 'tcp' then
1190 stdnse.debug1("http.request() supports the TCP protocol only, your request to %s cannot be completed.", host)
1191 return http_error("Unsupported protocol.")
1192 end
1193 end
1194
1195 method = string.match(data, "^(%S+)")
1196
1197 if type(host) == "string" and options.any_af then
1198 local status, addrs = nmap.resolve(host)
1199 host = addrs[1] or host
1200 end
1201
1202 local socket, partial, opts = comm.tryssl(host, port, data, { timeout = options.timeout })
1203
1204 if not socket then
1205 stdnse.debug1("http.request socket error: %s", partial)
1206 return http_error("Error creating socket.")
1207 end
1208
1209 stdnse.debug1(partial)
1210
1211 repeat
1212 response, partial = next_response(socket, method, partial)
1213
1214
1215
1216 if not response then
1217 return http_error("There was an error in next_response function.")
1218 end
1219 -- See RFC 2616, sections 8.2.3 and 10.1.1, for the 100 Continue status.
1220 -- Sometimes a server will tell us to "go ahead" with a POST body before
1221 -- sending the real response. If we got one of those, skip over it.
1222 until not (response.status >= 100 and response.status <= 199)
1223
1224 socket:close()
1225
1226 -- if SSL was used to retrieve the URL mark this in the response
1227 response.ssl = ( opts == 'ssl' )
1228
1229 return response
1230end
1231
1232---Do a single request with a given method. The response is returned as the standard
1233-- response table (see the module documentation).
1234--
1235-- The <code>get</code>, <code>head</code>, and <code>post</code> functions are simple
1236-- wrappers around <code>generic_request</code>.
1237--
1238-- Any 1XX (informational) responses are discarded.
1239--
1240-- @param host The host to connect to.
1241-- @param port The port to connect to.
1242-- @param method The method to use; for example, 'GET', 'HEAD', etc.
1243-- @param path The path to retrieve.
1244-- @param options [optional] A table that lets the caller control socket timeouts, HTTP headers, and other parameters. For full documentation, see the module documentation (above).
1245-- @return A response table, see module documentation for description.
1246-- @see request
1247function generic_request(host, port, method, path, options)
1248 stdnse.debug1("+++++++++++++++++++++++++++++=lololol=+++++++++++++++++++++++++++++")
1249 print("llllllllllllllllllllllllllllllllllll")
1250 if(not(validate_options(options))) then
1251 return http_error("Options failed to validate.")
1252 end
1253
1254 local digest_auth = options and options.auth and options.auth.digest
1255 local ntlm_auth = options and options.auth and options.auth.ntlm
1256
1257 if (digest_auth or ntlm_auth) and not have_ssl then
1258 stdnse.debug1("http: digest and ntlm auth require openssl.")
1259 end
1260
1261 if digest_auth and have_ssl then
1262 -- If we want to do digest authentication, we have to make an initial
1263 -- request to get realm, nonce and other fields.
1264 local options_with_auth_removed = tcopy(options)
1265 options_with_auth_removed["auth"] = nil
1266 local r = generic_request(host, port, method, path, options_with_auth_removed)
1267 local h = r.header['www-authenticate']
1268 if not r.status or (h and not string.find(h:lower(), "digest.-realm")) then
1269 stdnse.debug1("http: the target doesn't support digest auth or there was an error during request.")
1270 return http_error("The target doesn't support digest auth or there was an error during request.")
1271 end
1272 -- Compute the response hash
1273 local dmd5 = sasl.DigestMD5:new(h, options.auth.username, options.auth.password, method, path)
1274 local _, digest_table = dmd5:calcDigest()
1275 options.digestauth = digest_table
1276 end
1277
1278 if ntlm_auth and have_ssl then
1279
1280 local custom_options = tcopy(options) -- to be sent with the type 1 request
1281 custom_options["auth"] = nil -- removing the auth options
1282 -- let's check if the target supports ntlm with a simple get request.
1283 -- Setting a timeout here other than nil messes up the authentication if this is the first device sending
1284 -- a request to the server. Don't know why.
1285 custom_options.timeout = nil
1286 local response = generic_request(host, port, method, path, custom_options)
1287 local authentication_header = response.header['www-authenticate']
1288 -- get back the timeout option.
1289 custom_options.timeout = options.timeout
1290 custom_options.header = options.header or {}
1291 custom_options.header["Connection"] = "Keep-Alive" -- Keep-Alive headers are needed for authentication.
1292
1293 if (not authentication_header) or (not response.status) or (not string.find(authentication_header:lower(), "ntlm")) then
1294 stdnse.debug1("http: the target doesn't support NTLM or there was an error during request.")
1295 return http_error("The target doesn't support NTLM or there was an error during request.")
1296 end
1297
1298 -- ntlm works with three messages. we send a request, it sends
1299 -- a challenge, we respond to the challenge.
1300 local hostname = options.auth.hostname or "localhost" -- the hostname to be sent
1301 local workstation_name = options.auth.workstation_name or "NMAP" -- the workstation name to be sent
1302 local username = options.auth.username -- the username as specified
1303
1304 local auth_blob = "NTLMSSP\x00" .. -- NTLM signature
1305 "\x01\x00\x00\x00" .. -- NTLM Type 1 message
1306 bin.pack("<I", 0xa208b207) .. -- flags 56, 128, Version, Extended Security, Always Sign, Workstation supplied, Domain Supplied, NTLM Key, OEM, Unicode
1307 bin.pack("<SSISSI",#workstation_name, #workstation_name, 40 + #hostname, #hostname, #hostname, 40) .. -- Supplied Domain and Workstation
1308 bin.pack("CC<S", -- OS version info
1309 5, 1, 2600) .. -- 5.1.2600
1310 "\x00\x00\x00\x0f" .. -- OS version info end (static 0x0000000f)
1311 hostname.. -- HOST NAME
1312 workstation_name --WORKSTATION name
1313
1314 custom_options.ntlmauth = auth_blob
1315
1316 -- check if the protocol is tcp
1317 if type(port) == 'table' then
1318 if port.protocol and port.protocol ~= 'tcp' then
1319 stdnse.debug1("NTLM authentication supports the TCP protocol only, your request to %s cannot be completed.", host)
1320 return http_error("Unsupported protocol.")
1321 end
1322 end
1323
1324 -- tryssl uses ssl if needed. sends the type 1 message.
1325 local socket, partial, opts = comm.tryssl(host, port, build_request(host, port, method, path, custom_options), { timeout = options.timeout })
1326
1327 if not socket then
1328 return http_error("Could not create socket to send type 1 message.")
1329 end
1330
1331 repeat
1332 response, partial = next_response(socket, method, partial)
1333 if not response then
1334 return http_error("There was error in receiving response of type 1 message.")
1335 end
1336 until not (response.status >= 100 and response.status <= 199)
1337
1338 authentication_header = response.header['www-authenticate']
1339 -- take out the challenge
1340 local type2_response = authentication_header:sub(authentication_header:find(' ')+1, -1)
1341 local _, _, message_type, _, _, _, flags_received, challenge= bin.unpack("<A8ISSIIA8", base64.dec(type2_response))
1342 -- check if the response is a type 2 message.
1343 if message_type ~= 0x02 then
1344 stdnse.debug1("Expected type 2 message as response.")
1345 return
1346 end
1347
1348 local is_unicode = (bit.band(flags_received, 0x00000001) == 0x00000001) -- 0x00000001 UNICODE Flag
1349 local is_extended = (bit.band(flags_received, 0x00080000) == 0x00080000) -- 0x00080000 Extended Security Flag
1350 local type_3_flags = 0xa2888206 -- flags 56, 128, Version, Target Info, Extended Security, Always Sign, NTLM Key, OEM
1351
1352 local lanman, ntlm
1353 if is_extended then
1354 -- this essentially calls the new ntlmv2_session_response function in smbauth.lua and returns whatever it returns
1355 lanman, ntlm = smbauth.get_password_response(nil, username, "", options.auth.password, nil, "ntlmv2_session", challenge, true)
1356 else
1357 lanman, ntlm = smbauth.get_password_response(nil, username, "", options.auth.password, nil, "ntlm", challenge, false)
1358 type_3_flags = type_3_flags - 0x00080000 -- Removing the Extended Security Flag as server doesn't support it.
1359 end
1360
1361 local domain = ""
1362 local session_key = ""
1363
1364 -- if server supports unicode, then strings are sent in unicode format.
1365 if is_unicode then
1366 username = unicode.utf8to16(username)
1367 hostname = unicode.utf8to16(hostname)
1368 type_3_flags = type_3_flags - 0x00000001 -- OEM flag is 0x00000002. removing 0x00000001 results in UNICODE flag.
1369 end
1370
1371 local BASE_OFFSET = 72 -- Version 3 -- The Session Key<empty in our case>, flags, and OS Version structure are all present.
1372
1373 auth_blob = bin.pack("<zISSISSISSISSISSISSIICCSAAAAA",
1374 "NTLMSSP",
1375 0x00000003,
1376 #lanman,
1377 #lanman,
1378 BASE_OFFSET + #username + #hostname,
1379 ( #ntlm ),
1380 ( #ntlm ),
1381 BASE_OFFSET + #username + #hostname + #lanman,
1382 #domain,
1383 #domain,
1384 BASE_OFFSET,
1385 #username,
1386 #username,
1387 BASE_OFFSET,
1388 #hostname,
1389 #hostname,
1390 BASE_OFFSET + #username,
1391 #session_key,
1392 #session_key,
1393 BASE_OFFSET + #username + #hostname + #lanman + #ntlm,
1394 type_3_flags,
1395 5,
1396 1,
1397 2600,
1398 "\x00\x00\x00\x0f",
1399 username,
1400 hostname,
1401 lanman,
1402 ntlm)
1403
1404 custom_options.ntlmauth = auth_blob
1405 socket:send(build_request(host, port, method, path, custom_options))
1406
1407 repeat
1408 response, partial = next_response(socket, method, partial)
1409 if not response then
1410 return http_error("There was error in receiving response of type 3 message.")
1411 end
1412 until not (response.status >= 100 and response.status <= 199)
1413
1414 socket:close()
1415 response.ssl = ( opts == 'ssl' )
1416
1417 return response
1418 end
1419
1420 return request(host, port, build_request(host, port, method, path, options), options)
1421end
1422
1423---Uploads a file using the PUT method and returns a result table. This is a simple wrapper
1424-- around <code>generic_request</code>
1425--
1426-- @param host The host to connect to.
1427-- @param port The port to connect to.
1428-- @param path The path to retrieve.
1429-- @param options [optional] A table that lets the caller control socket timeouts, HTTP headers, and other parameters. For full documentation, see the module documentation (above).
1430-- @param putdata The contents of the file to upload
1431-- @return A response table, see module documentation for description.
1432-- @see http.generic_request
1433function put(host, port, path, options, putdata)
1434 if(not(validate_options(options))) then
1435 return http_error("Options failed to validate.")
1436 end
1437 if ( not(putdata) ) then
1438 return http_error("No file to PUT.")
1439 end
1440 local mod_options = {
1441 content = putdata,
1442 }
1443 table_augment(mod_options, options or {})
1444 return generic_request(host, port, "PUT", path, mod_options)
1445end
1446
1447-- A battery of tests a URL is subjected to in order to decide if it may be
1448-- redirected to. They incrementally fill in loc.host, loc.port, and loc.path.
1449local redirect_ok_rules = {
1450
1451 -- Check if there's any credentials in the url
1452 function (url, host, port)
1453 -- bail if userinfo is present
1454 return ( url.userinfo and false ) or true
1455 end,
1456
1457 -- Check if the location is within the domain or host
1458 function (url, host, port)
1459 local hostname = stdnse.get_hostname(host)
1460 if ( hostname == host.ip and host.ip == url.host.ip ) then
1461 return true
1462 end
1463 local domain = hostname:match("^[^%.]-%.(.*)") or hostname
1464 local match = ("^.*%s$"):format(domain)
1465 if ( url.host:match(match) ) then
1466 return true
1467 end
1468 return false
1469 end,
1470
1471 -- Check whether the new location has the same port number
1472 function (url, host, port)
1473 -- port fixup, adds default ports 80 and 443 in case no url.port was
1474 -- defined, we do this based on the url scheme
1475 local url_port = url.port
1476 if ( not(url_port) ) then
1477 if ( url.scheme == "http" ) then
1478 url_port = 80
1479 elseif( url.scheme == "https" ) then
1480 url_port = 443
1481 end
1482 end
1483 if (not url_port) or tonumber(url_port) == port.number then
1484 return true
1485 end
1486 return false
1487 end,
1488
1489 -- Check whether the url.scheme matches the port.service
1490 function (url, host, port)
1491 -- if url.scheme is present then it must match the scanned port
1492 if url.scheme and url.port then return true end
1493 if url.scheme and url.scheme ~= port.service then return false end
1494 return true
1495 end,
1496
1497 -- make sure we're actually being redirected somewhere and not to the same url
1498 function (url, host, port)
1499 -- path cannot be unchanged unless host has changed
1500 -- loc.path must be set if returning true
1501 if ( not url.path or url.path == "/" ) and url.host == ( host.targetname or host.ip) then return false end
1502 if not url.path then return true end
1503 return true
1504 end,
1505}
1506
1507--- Check if the given URL is okay to redirect to. Return a table with keys
1508-- "host", "port", and "path" if okay, nil otherwise.
1509--
1510-- Redirects will be followed unless they:
1511-- * contain credentials
1512-- * are on a different domain or host
1513-- * have a different port number or URI scheme
1514-- * redirect to the same URI
1515-- * exceed the maximum number of redirects specified
1516-- @param url table as returned by url.parse
1517-- @param host table as received by the action function
1518-- @param port table as received by the action function
1519-- @param counter number of redirects to follow.
1520-- @return loc table containing the new location
1521function redirect_ok(host, port, counter)
1522 -- convert a numeric port to a table
1523 if ( "number" == type(port) ) then
1524 port = { number = port }
1525 end
1526 return function(url)
1527 if ( counter == 0 ) then return false end
1528 counter = counter - 1
1529 for i, rule in ipairs( redirect_ok_rules ) do
1530 if ( not(rule( url, host, port )) ) then
1531 --stdnse.debug1("Rule failed: %d", i)
1532 return false
1533 end
1534 end
1535 return true
1536 end
1537end
1538
1539--- Handles a HTTP redirect
1540-- @param host table as received by the script action function
1541-- @param port table as received by the script action function
1542-- @param path string
1543-- @param response table as returned by http.get or http.head
1544-- @return url table as returned by <code>url.parse</code> or nil if there's no
1545-- redirect taking place
1546function parse_redirect(host, port, path, response)
1547 if ( not(tostring(response.status):match("^30[01237]$")) or
1548 not(response.header) or
1549 not(response.header.location) ) then
1550 return nil
1551 end
1552 port = ( "number" == type(port) ) and { number = port } or port
1553 local u = url.parse(response.header.location)
1554 if ( not(u.host) ) then
1555 -- we're dealing with a relative url
1556 u.host = stdnse.get_hostname(host)
1557 u.path = ((u.path:sub(1,1) == "/" and "" ) or "/" ) .. u.path -- ensuring leading slash
1558 end
1559 -- do port fixup
1560 if ( not(u.port) ) then
1561 if ( u.scheme == "http" ) then u.port = 80
1562 elseif ( u.scheme == "https") then u.port = 443
1563 else u.port = port.number end
1564 end
1565 if ( not(u.path) ) then
1566 u.path = "/"
1567 end
1568 if ( u.query ) then
1569 u.path = ("%s?%s"):format( u.path, u.query )
1570 end
1571 return u
1572end
1573
1574-- Retrieves the correct function to use to validate HTTP redirects
1575-- @param host table as received by the action function
1576-- @param port table as received by the action function
1577-- @param options table as passed to http.get or http.head
1578-- @return redirect_ok function used to validate HTTP redirects
1579local function get_redirect_ok(host, port, options)
1580 if ( options ) then
1581 if ( options.redirect_ok == false ) then
1582 return function() return false end
1583 elseif( "function" == type(options.redirect_ok) ) then
1584 return options.redirect_ok(host, port)
1585 elseif( type(options.redirect_ok) == "number") then
1586 return redirect_ok(host, port, options.redirect_ok)
1587 else
1588 return redirect_ok(host, port, MAX_REDIRECT_COUNT)
1589 end
1590 else
1591 return redirect_ok(host, port, MAX_REDIRECT_COUNT)
1592 end
1593end
1594
1595---Fetches a resource with a GET request and returns the result as a table.
1596--
1597-- This is a simple wrapper around <code>generic_request</code>, with the added
1598-- benefit of having local caching and support for HTTP redirects. Redirects
1599-- are followed only if they pass all the validation rules of the redirect_ok
1600-- function. This function may be overridden by supplying a custom function in
1601-- the <code>redirect_ok</code> field of the options array. The default
1602-- function redirects the request if the destination is:
1603-- * Within the same host or domain
1604-- * Has the same port number
1605-- * Stays within the current scheme
1606-- * Does not exceed <code>MAX_REDIRECT_COUNT</code> count of redirects
1607--
1608-- Caching and redirects can be controlled in the <code>options</code> array,
1609-- see module documentation for more information.
1610--
1611-- @param host The host to connect to.
1612-- @param port The port to connect to.
1613-- @param path The path to retrieve.
1614-- @param options [optional] A table that lets the caller control socket
1615-- timeouts, HTTP headers, and other parameters. For full
1616-- documentation, see the module documentation (above).
1617-- @return A response table, see module documentation for description.
1618-- @see http.generic_request
1619function get(host, port, path, options)
1620 if(not(validate_options(options))) then
1621 return http_error("Options failed to validate.")
1622 end
1623 local redir_check = get_redirect_ok(host, port, options)
1624 local response, state, location
1625 local u = { host = host, port = port, path = path }
1626 repeat
1627 response, state = lookup_cache("GET", u.host, u.port, u.path, options);
1628 if ( response == nil ) then
1629 response = generic_request(u.host, u.port, "GET", u.path, options)
1630 insert_cache(state, response);
1631 end
1632 u = parse_redirect(host, port, path, response)
1633 if ( not(u) ) then
1634 break
1635 end
1636 location = location or {}
1637 table.insert(location, response.header.location)
1638 until( not(redir_check(u)) )
1639 response.location = location
1640 return response
1641end
1642
1643---Parses a URL and calls <code>http.get</code> with the result. The URL can contain
1644-- all the standard fields, protocol://host:port/path
1645--
1646-- @param u The URL of the host.
1647-- @param options [optional] A table that lets the caller control socket timeouts, HTTP headers, and other parameters. For full documentation, see the module documentation (above).
1648-- @return A response table, see module documentation for description.
1649-- @see http.get
1650function get_url( u, options )
1651 if(not(validate_options(options))) then
1652 return http_error("Options failed to validate.")
1653 end
1654 local parsed = url.parse( u )
1655 local port = {}
1656
1657 port.service = parsed.scheme
1658 port.number = parsed.port
1659
1660 if not port.number then
1661 if parsed.scheme == 'https' then
1662 port.number = 443
1663 else
1664 port.number = 80
1665 end
1666 end
1667
1668 local path = parsed.path or "/"
1669 if parsed.query then
1670 path = path .. "?" .. parsed.query
1671 end
1672
1673 return get( parsed.host, port, path, options )
1674end
1675
1676---Fetches a resource with a HEAD request.
1677--
1678-- Like <code>get</code>, this is a simple wrapper around
1679-- <code>generic_request</code> with response caching. This function also has
1680-- support for HTTP redirects. Redirects are followed only if they pass all the
1681-- validation rules of the redirect_ok function. This function may be
1682-- overridden by supplying a custom function in the <code>redirect_ok</code>
1683-- field of the options array. The default function redirects the request if
1684-- the destination is:
1685-- * Within the same host or domain
1686-- * Has the same port number
1687-- * Stays within the current scheme
1688-- * Does not exceed <code>MAX_REDIRECT_COUNT</code> count of redirects
1689--
1690-- Caching and redirects can be controlled in the <code>options</code> array,
1691-- see module documentation for more information.
1692--
1693-- @param host The host to connect to.
1694-- @param port The port to connect to.
1695-- @param path The path to retrieve.
1696-- @param options [optional] A table that lets the caller control socket
1697-- timeouts, HTTP headers, and other parameters. For full
1698-- documentation, see the module documentation (above).
1699-- @return A response table, see module documentation for description.
1700-- @see http.generic_request
1701function head(host, port, path, options)
1702 if(not(validate_options(options))) then
1703 return http_error("Options failed to validate.")
1704 end
1705 local redir_check = get_redirect_ok(host, port, options)
1706 local response, state, location
1707 local u = { host = host, port = port, path = path }
1708 repeat
1709 response, state = lookup_cache("HEAD", u.host, u.port, u.path, options);
1710 if response == nil then
1711 response = generic_request(u.host, u.port, "HEAD", u.path, options)
1712 insert_cache(state, response);
1713 end
1714 u = parse_redirect(host, port, path, response)
1715 if ( not(u) ) then
1716 break
1717 end
1718 location = location or {}
1719 table.insert(location, response.header.location)
1720 until( not(redir_check(u)) )
1721 response.location = location
1722 return response
1723end
1724
1725---Fetches a resource with a POST request.
1726--
1727-- Like <code>get</code>, this is a simple wrapper around
1728-- <code>generic_request</code> except that postdata is handled properly.
1729--
1730-- @param host The host to connect to.
1731-- @param port The port to connect to.
1732-- @param path The path to retrieve.
1733-- @param options [optional] A table that lets the caller control socket
1734-- timeouts, HTTP headers, and other parameters. For full
1735-- documentation, see the module documentation (above).
1736-- @param ignored Ignored for backwards compatibility.
1737-- @param postdata A string or a table of data to be posted. If a table, the
1738-- keys and values must be strings, and they will be encoded
1739-- into an application/x-www-form-encoded form submission.
1740-- @return A response table, see module documentation for description.
1741-- @see http.generic_request
1742function post( host, port, path, options, ignored, postdata )
1743 if(not(validate_options(options))) then
1744 return http_error("Options failed to validate.")
1745 end
1746 local mod_options = {
1747 content = postdata,
1748 }
1749 table_augment(mod_options, options or {})
1750 return generic_request(host, port, "POST", path, mod_options)
1751end
1752
1753-- Deprecated pipeline functions
1754function pGet( host, port, path, options, ignored, allReqs )
1755 stdnse.debug1("WARNING: pGet() is deprecated. Use pipeline_add() instead.")
1756 return pipeline_add(path, options, allReqs, 'GET')
1757end
1758function pHead( host, port, path, options, ignored, allReqs )
1759 stdnse.debug1("WARNING: pHead() is deprecated. Use pipeline_add instead.")
1760 return pipeline_add(path, options, allReqs, 'HEAD')
1761end
1762function addPipeline(host, port, path, options, ignored, allReqs, method)
1763 stdnse.debug1("WARNING: addPipeline() is deprecated! Use pipeline_add instead.")
1764 return pipeline_add(path, options, allReqs, method)
1765end
1766function pipeline(host, port, allReqs)
1767 stdnse.debug1("WARNING: pipeline() is deprecated. Use pipeline_go() instead.")
1768 return pipeline_go(host, port, allReqs)
1769end
1770
1771
1772---Adds a pending request to the HTTP pipeline.
1773--
1774-- The HTTP pipeline is a set of requests that will all be sent at the same
1775-- time, or as close as the server allows. This allows more efficient code,
1776-- since requests are automatically buffered and sent simultaneously.
1777--
1778-- The <code>all_requests</code> argument contains the current list of queued
1779-- requests (if this is the first time calling <code>pipeline_add</code>, it
1780-- should be <code>nil</code>). After adding the request to end of the queue,
1781-- the queue is returned and can be passed to the next
1782-- <code>pipeline_add</code> call.
1783--
1784-- When all requests have been queued, call <code>pipeline_go</code> with the
1785-- all_requests table that has been built.
1786--
1787-- @param path The path to retrieve.
1788-- @param options [optional] A table that lets the caller control socket
1789-- timeouts, HTTP headers, and other parameters. For full
1790-- documentation, see the module documentation (above).
1791-- @param all_requests [optional] The current pipeline queue (returned from a
1792-- previous <code>add_pipeline</code> call), or nil if it's
1793-- the first call.
1794-- @param method [optional] The HTTP method ('GET', 'HEAD', 'POST', etc).
1795-- Default: 'GET'.
1796-- @return Table with the pipeline requests (plus this new one)
1797-- @see http.pipeline_go
1798function pipeline_add(path, options, all_requests, method)
1799 if(not(validate_options(options))) then
1800 return nil
1801 end
1802 method = method or 'GET'
1803 all_requests = all_requests or {}
1804
1805 local mod_options = {
1806 header = {
1807 ["Connection"] = "keep-alive"
1808 }
1809 }
1810 table_augment(mod_options, options or {})
1811
1812 local object = { method=method, path=path, options=mod_options }
1813 table.insert(all_requests, object)
1814
1815 return all_requests
1816end
1817
1818---Performs all queued requests in the all_requests variable (created by the
1819-- <code>pipeline_add</code> function).
1820--
1821-- Returns an array of responses, each of which is a table as defined in the
1822-- module documentation above.
1823--
1824-- @param host The host to connect to.
1825-- @param port The port to connect to.
1826-- @param all_requests A table with all the previously built pipeline requests
1827-- @return A list of responses, in the same order as the requests were queued.
1828-- Each response is a table as described in the module documentation.
1829-- The response list may be either nil or shorter than expected (up to
1830-- and including being completely empty) due to communication issues.
1831function pipeline_go(host, port, all_requests)
1832 local responses
1833 local response
1834 local partial
1835
1836 responses = {}
1837
1838 -- Check for an empty request
1839 if (not all_requests or #all_requests == 0) then
1840 stdnse.debug1("Warning: empty set of requests passed to http.pipeline_go()")
1841 return responses
1842 end
1843 stdnse.debug1("Total number of pipelined requests: " .. #all_requests)
1844
1845 local socket, bopt
1846
1847 -- We'll try a first request with keep-alive, just to check if the server
1848 -- supports and how many requests we can send into one socket!
1849 local request = build_request(host, port, all_requests[1].method, all_requests[1].path, all_requests[1].options)
1850
1851 socket, partial, bopt = comm.tryssl(host, port, request, {recv_before=false})
1852 if not socket then
1853 return nil
1854 end
1855
1856 response, partial = next_response(socket, all_requests[1].method, partial)
1857 if not response then
1858 return nil
1859 end
1860
1861 table.insert(responses, response)
1862
1863 local limit = getPipelineMax(response) -- how many requests to send on one connection
1864 limit = limit > #all_requests and #all_requests or limit
1865 local max_pipeline = stdnse.get_script_args("http.max-pipeline") or limit -- how many requests should be pipelined
1866 local count = 1
1867 stdnse.debug1("Number of requests allowed by pipeline: " .. limit)
1868
1869 while #responses < #all_requests do
1870 local j, batch_end
1871 -- we build a table with many requests, upper limited by the var "limit"
1872 local requests = {}
1873
1874 if #responses + limit < #all_requests then
1875 batch_end = #responses + limit
1876 else
1877 batch_end = #all_requests
1878 end
1879
1880 j = #responses + 1
1881 while j <= batch_end do
1882 if j == batch_end then
1883 all_requests[j].options.header["Connection"] = "close"
1884 end
1885 if j~= batch_end and all_requests[j].options.header["Connection"] ~= 'keep-alive' then
1886 all_requests[j].options.header["Connection"] = 'keep-alive'
1887 end
1888 table.insert(requests, build_request(host, port, all_requests[j].method, all_requests[j].path, all_requests[j].options))
1889 -- to avoid calling build_request more than one time on the same request,
1890 -- we might want to build all the requests once, above the main while loop
1891 j = j + 1
1892 end
1893
1894 if count >= limit or not socket:get_info() then
1895 socket:connect(host, port, bopt)
1896 partial = ""
1897 count = 0
1898 end
1899 socket:set_timeout(10000)
1900
1901 local start = 1
1902 local len = #requests
1903 local req_sent = 0
1904 -- start sending the requests and pipeline them in batches of max_pipeline elements
1905 while start <= len do
1906 stdnse.debug2("HTTP pipeline: number of requests in current batch: %d, already sent: %d, responses from current batch: %d, all responses received: %d",len,start-1,count,#responses)
1907 local req = {}
1908 if max_pipeline == limit then
1909 req = requests
1910 else
1911 for i=start,start+max_pipeline-1,1 do
1912 table.insert(req, requests[i])
1913 end
1914 end
1915 local num_req = #req
1916 req = table.concat(req, "")
1917 start = start + max_pipeline
1918 socket:send(req)
1919 req_sent = req_sent + num_req
1920 local inner_count = 0
1921 local fail = false
1922 -- collect responses for the last batch
1923 while inner_count < num_req and #responses < #all_requests do
1924 response, partial = next_response(socket, all_requests[#responses + 1].method, partial)
1925 if not response then
1926 stdnse.debug1("HTTP pipeline: there was a problem while receiving responses.")
1927 stdnse.debug3("The request was:\n%s",req)
1928 fail = true
1929 break
1930 end
1931 count = count + 1
1932 inner_count = inner_count + 1
1933 responses[#responses + 1] = response
1934 end
1935 if fail then break end
1936 end
1937
1938 socket:close()
1939
1940 if count == 0 then
1941 stdnse.debug1("Received 0 of %d expected responses.\nGiving up on pipeline.", limit);
1942 break
1943 elseif count < req_sent then
1944 stdnse.debug1("Received only %d of %d expected responses.\nDecreasing max pipelined requests to %d.", count, req_sent, count)
1945 limit = count
1946 end
1947 end
1948
1949 stdnse.debug1("Number of received responses: " .. #responses)
1950
1951 return responses
1952end
1953
1954
1955-- Parsing of specific headers. skip_space and the read_* functions return the
1956-- byte index following whatever they have just read, or nil on error.
1957
1958-- Skip whitespace (that has already been folded from LWS). See RFC 2616,
1959-- section 2.2, definition of LWS.
1960local function skip_space(s, pos)
1961 local _
1962
1963 _, pos = string.find(s, "^[ \t]*", pos)
1964
1965 return pos + 1
1966end
1967
1968-- See RFC 2616, section 2.2.
1969local function read_token(s, pos)
1970 local _, token
1971
1972 pos = skip_space(s, pos)
1973 -- 1*<any CHAR except CTLs or separators>. CHAR is only byte values 0-127.
1974 _, pos, token = string.find(s, "^([^\0\001-\031()<>@,;:\\\"/?={} \t%[%]\127-\255]+)", pos)
1975
1976 if token then
1977 return pos + 1, token
1978 else
1979 return nil
1980 end
1981end
1982
1983-- See RFC 2616, section 2.2. Here we relax the restriction that TEXT may not
1984-- contain CTLs.
1985local function read_quoted_string(s, pos)
1986 local chars = {}
1987
1988 if string.sub(s, pos, pos) ~= "\"" then
1989 return nil
1990 end
1991 pos = pos + 1
1992 pos = skip_space(s, pos)
1993 while pos <= #s and string.sub(s, pos, pos) ~= "\"" do
1994 local c
1995
1996 c = string.sub(s, pos, pos)
1997 if c == "\\" then
1998 if pos < #s then
1999 pos = pos + 1
2000 c = string.sub(s, pos, pos)
2001 else
2002 return nil
2003 end
2004 end
2005
2006 chars[#chars + 1] = c
2007 pos = pos + 1
2008 end
2009 if pos > #s or string.sub(s, pos, pos) ~= "\"" then
2010 return nil
2011 end
2012
2013 return pos + 1, table.concat(chars)
2014end
2015
2016local function read_token_or_quoted_string(s, pos)
2017 pos = skip_space(s, pos)
2018 if string.sub(s, pos, pos) == "\"" then
2019 return read_quoted_string(s, pos)
2020 else
2021 return read_token(s, pos)
2022 end
2023end
2024
2025--- Create a pattern to find a tag
2026--
2027-- Case-insensitive search for tags
2028-- @param tag The name of the tag to find
2029-- @param endtag Boolean true if you are looking for an end tag, otherwise it will look for a start tag
2030-- @return A pattern to find the tag
2031function tag_pattern(tag, endtag)
2032 local patt = {}
2033 if endtag then
2034 patt[1] = "</%s*"
2035 else
2036 patt[1] = "<%s*"
2037 end
2038 local up, down = tag:upper(), tag:lower()
2039 for i = 1, #tag do
2040 patt[#patt+1] = string.format("[%s%s]", up:sub(i,i), down:sub(i,i))
2041 end
2042 if endtag then
2043 patt[#patt+1] = "%f[%s>].->"
2044 else
2045 patt[#patt+1] = "%f[%s/>].->"
2046 end
2047 return table.concat(patt)
2048end
2049
2050---
2051-- Finds forms in html code
2052--
2053-- returns table of found forms, in plaintext.
2054-- @param body A <code>response.body</code> in which to search for forms
2055-- @return A list of forms.
2056function grab_forms(body)
2057 local forms = {}
2058 if not body then return forms end
2059 local form_start_expr = tag_pattern("form")
2060 local form_end_expr = tag_pattern("form", true)
2061
2062 local form_opening = string.find(body, form_start_expr)
2063 local forms = {}
2064
2065 while form_opening do
2066 local form_closing = string.find(body, form_end_expr, form_opening+1)
2067 if form_closing == nil then --html code contains errors
2068 break
2069 end
2070 forms[#forms+1] = string.sub(body, form_opening, form_closing-1)
2071 if form_closing+1 <= #body then
2072 form_opening = string.find(body, form_start_expr, form_closing+1)
2073 else
2074 break
2075 end
2076 end
2077 return forms
2078end
2079
2080local function get_attr (html, name)
2081 local lhtml = html:lower()
2082 local lname = name:lower()
2083 -- try the attribute-value syntax first
2084 local _, pos = lhtml:find('%s' .. lname .. '%s*=%s*[^%s]')
2085 if not pos then
2086 -- try the empty attribute syntax and, if found,
2087 -- return zero-length string as its value; nil otherwise
2088 return lhtml:match('[^%s=]%s+' .. lname .. '[%s/>]') and "" or nil
2089 end
2090 local value
2091 _, value = html:match('^([\'"])(.-)%1', pos)
2092 if not value then
2093 value = html:match('^[^%s<>=\'"`]+', pos)
2094 end
2095 return slaxml.parser.unescape(value)
2096end
2097---
2098-- Parses a form, that is, finds its action and fields.
2099-- @param form A plaintext representation of form
2100-- @return A dictionary with keys: <code>action</code>,
2101-- <code>method</code> if one is specified, <code>fields</code>
2102-- which is a list of fields found in the form each of which has a
2103-- <code>name</code> attribute and <code>type</code> if specified.
2104function parse_form(form)
2105 local parsed = {}
2106 local fields = {}
2107 local form_action = get_attr(form, "action")
2108 if form_action then
2109 parsed["action"] = form_action
2110 end
2111
2112 -- determine if the form is using get or post
2113 local form_method = get_attr(form, "method")
2114 if form_method then
2115 parsed["method"] = string.lower(form_method)
2116 end
2117
2118 -- get the id of the form
2119 local form_id = get_attr(form, "id")
2120 if form_id then
2121 parsed["id"] = string.lower(form_id)
2122 end
2123
2124 -- now identify the fields
2125 local input_type
2126 local input_name
2127 local input_value
2128
2129 -- first find regular inputs
2130 for f in string.gmatch(form, tag_pattern("input")) do
2131 input_type = get_attr(f, "type")
2132 input_name = get_attr(f, "name")
2133 input_value = get_attr(f, "value")
2134 local next_field_index = #fields+1
2135 if input_name then
2136 fields[next_field_index] = {}
2137 fields[next_field_index]["name"] = input_name
2138 if input_type then
2139 fields[next_field_index]["type"] = string.lower(input_type)
2140 end
2141 if input_value then
2142 fields[next_field_index]["value"] = input_value
2143 end
2144 end
2145 end
2146
2147 -- now search for textareas
2148 for f in string.gmatch(form, tag_pattern("textarea")) do
2149 input_name = get_attr(f, "name")
2150 local next_field_index = #fields+1
2151 if input_name then
2152 fields[next_field_index] = {}
2153 fields[next_field_index]["name"] = input_name
2154 fields[next_field_index]["type"] = "textarea"
2155 end
2156 end
2157 parsed["fields"] = fields
2158 return parsed
2159end
2160
2161local MONTH_MAP = {
2162 Jan = 1, Feb = 2, Mar = 3, Apr = 4, May = 5, Jun = 6,
2163 Jul = 7, Aug = 8, Sep = 9, Oct = 10, Nov = 11, Dec = 12
2164}
2165
2166--- Parses an HTTP date string
2167--
2168-- Supports any of the following formats from section 3.3.1 of RFC 2616:
2169-- * Sun, 06 Nov 1994 08:49:37 GMT (RFC 822, updated by RFC 1123)
2170-- * Sunday, 06-Nov-94 08:49:37 GMT (RFC 850, obsoleted by RFC 1036)
2171-- * Sun Nov 6 08:49:37 1994 (ANSI C's <code>asctime()</code> format)
2172-- @param s the date string.
2173-- @return a table with keys <code>year</code>, <code>month</code>,
2174-- <code>day</code>, <code>hour</code>, <code>min</code>, <code>sec</code>, and
2175-- <code>isdst</code>, relative to GMT, suitable for input to
2176-- <code>os.time</code>.
2177function parse_date(s)
2178 local day, month, year, hour, min, sec, tz, month_name
2179
2180 -- Handle RFC 1123 and 1036 at once.
2181 day, month_name, year, hour, min, sec, tz = s:match("^%w+, (%d+)[- ](%w+)[- ](%d+) (%d+):(%d+):(%d+) (%w+)$")
2182 if not day then
2183 month_name, day, hour, min, sec, year = s:match("%w+ (%w+) ?(%d+) (%d+):(%d+):(%d+) (%d+)")
2184 tz = "GMT"
2185 end
2186 if not day then
2187 stdnse.debug1("http.parse_date: can't parse date \"%s\": unknown format.", s)
2188 return nil
2189 end
2190 -- Look up the numeric code for month.
2191 month = MONTH_MAP[month_name]
2192 if not month then
2193 stdnse.debug1("http.parse_date: unknown month name \"%s\".", month_name)
2194 return nil
2195 end
2196 if tz ~= "GMT" then
2197 stdnse.debug1("http.parse_date: don't know time zone \"%s\", only \"GMT\".", tz)
2198 return nil
2199 end
2200 day = tonumber(day)
2201 year = tonumber(year)
2202 hour = tonumber(hour)
2203 min = tonumber(min)
2204 sec = tonumber(sec)
2205
2206 if year < 100 then
2207 -- Two-digit year. Make a guess.
2208 if year < 70 then
2209 year = year + 2000
2210 else
2211 year = year + 1900
2212 end
2213 end
2214
2215 return { year = year, month = month, day = day, hour = hour, min = min, sec = sec, isdst = false }
2216end
2217
2218-- See RFC 2617, section 1.2. This function returns a table with keys "scheme"
2219-- and "params".
2220local function read_auth_challenge(s, pos)
2221 local _, scheme, params
2222
2223 pos, scheme = read_token(s, pos)
2224 if not scheme then
2225 return nil
2226 end
2227
2228 params = {}
2229 pos = skip_space(s, pos)
2230 while pos < #s do
2231 local name, val
2232 local tmp_pos
2233
2234 -- We need to peek ahead at this point. It's possible that we've hit the
2235 -- end of one challenge and the beginning of another. Section 14.33 says
2236 -- that the header value can be 1#challenge, in other words several
2237 -- challenges separated by commas. Because the auth-params are also
2238 -- separated by commas, the only way we can tell is if we find a token not
2239 -- followed by an equals sign.
2240 tmp_pos = pos
2241 tmp_pos, name = read_token(s, tmp_pos)
2242 if not name then
2243 pos = skip_space(s, pos + 1)
2244 return pos, { scheme = scheme, params = nil }
2245 end
2246 tmp_pos = skip_space(s, tmp_pos)
2247 if string.sub(s, tmp_pos, tmp_pos) ~= "=" then
2248 -- No equals sign, must be the beginning of another challenge.
2249 break
2250 end
2251 tmp_pos = tmp_pos + 1
2252
2253 pos = tmp_pos
2254 pos, val = read_token_or_quoted_string(s, pos)
2255 if not val then
2256 return nil
2257 end
2258 if params[name] then
2259 return nil
2260 end
2261 params[name] = val
2262 pos = skip_space(s, pos)
2263 if string.sub(s, pos, pos) == "," then
2264 pos = skip_space(s, pos + 1)
2265 if pos > #s then
2266 return nil
2267 end
2268 end
2269 end
2270
2271 return pos, { scheme = scheme, params = params }
2272end
2273
2274---Parses the WWW-Authenticate header as described in RFC 2616, section 14.47
2275-- and RFC 2617, section 1.2.
2276--
2277-- The return value is an array of challenges. Each challenge is a table with
2278-- the keys <code>scheme</code> and <code>params</code>.
2279-- @param s The header value text.
2280-- @return An array of challenges, or <code>nil</code> on error.
2281function parse_www_authenticate(s)
2282 local challenges = {}
2283 local pos
2284
2285 pos = 1
2286 while pos <= #s do
2287 local challenge
2288
2289 pos, challenge = read_auth_challenge(s, pos)
2290 if not challenge then
2291 return nil
2292 end
2293 challenges[#challenges + 1] = challenge
2294 end
2295
2296 return challenges
2297end
2298
2299
2300---Take the data returned from a HTTP request and return the status string.
2301-- Useful for <code>stdnse.debug</code> messages and even advanced output.
2302--
2303-- @param data The response table from any HTTP request
2304-- @return The best status string we could find: either the actual status string, the status code, or <code>"<unknown status>"</code>.
2305function get_status_string(data)
2306 -- Make sure we have valid data
2307 if(data == nil) then
2308 return "<unknown status>"
2309 elseif(data['status-line'] == nil) then
2310 if(data['status'] ~= nil) then
2311 return data['status']
2312 end
2313
2314 return "<unknown status>"
2315 end
2316
2317 -- We basically want everything after the space
2318 local space = string.find(data['status-line'], ' ')
2319 if(space == nil) then
2320 return data['status-line']
2321 else
2322 return (string.sub(data['status-line'], space + 1)):gsub('\r?\n', '')
2323 end
2324end
2325
2326---Determine whether or not the server supports HEAD.
2327--
2328-- Tests by requesting / and verifying that it returns 200, and doesn't return
2329-- data. We implement the check like this because can't always rely on OPTIONS
2330-- to tell the truth.
2331--
2332-- Note: If <code>identify_404</code> returns a 200 status, HEAD requests
2333-- should be disabled. Sometimes, servers use a 200 status code with a message
2334-- explaining that the page wasn't found. In this case, to actually identify
2335-- a 404 page, we need the full body that a HEAD request doesn't supply.
2336-- This is determined automatically if the <code>result_404</code> field is
2337-- set.
2338--
2339-- @param host The host object.
2340-- @param port The port to use.
2341-- @param result_404 [optional] The result when an unknown page is requested.
2342-- This is returned by <code>identify_404</code>. If the 404
2343-- page returns a 200 code, then we disable HEAD requests.
2344-- @param path The path to request; by default, / is used.
2345-- @return A boolean value: true if HEAD is usable, false otherwise.
2346-- @return If HEAD is usable, the result of the HEAD request is returned (so
2347-- potentially, a script can avoid an extra call to HEAD)
2348function can_use_head(host, port, result_404, path)
2349 -- If the 404 result is 200, don't use HEAD.
2350 if(result_404 == 200) then
2351 return false
2352 end
2353
2354 -- Default path
2355 if(path == nil) then
2356 path = '/'
2357 end
2358
2359 -- Perform a HEAD request and see what happens.
2360 local data = head( host, port, path )
2361 if data then
2362 if data.status and data.status == 302 and data.header and data.header.location then
2363 stdnse.debug1("HTTP: Warning: Host returned 302 and not 200 when performing HEAD.")
2364 return false
2365 end
2366
2367 if data.status and data.status == 200 and data.header then
2368 -- check that a body wasn't returned
2369 if #data.body > 0 then
2370 stdnse.debug1("HTTP: Warning: Host returned data when performing HEAD.")
2371 return false
2372 end
2373
2374 stdnse.debug1("HTTP: Host supports HEAD.")
2375 return true, data
2376 end
2377
2378 stdnse.debug1("HTTP: Didn't receive expected response to HEAD request (got %s).", get_status_string(data))
2379 return false
2380 end
2381
2382 stdnse.debug1("HTTP: HEAD request completely failed.")
2383 return false
2384end
2385
2386--- Try to remove anything that might change within a 404.
2387--
2388-- For example:
2389-- * A file path (includes URI)
2390-- * A time
2391-- * A date
2392-- * An execution time (numbers in general, really)
2393--
2394-- The intention is that two 404 pages from different URIs and taken hours
2395-- apart should, whenever possible, look the same.
2396--
2397-- During this function, we're likely going to over-trim things. This is fine
2398-- -- we want enough to match on that it'll a) be unique, and b) have the best
2399-- chance of not changing. Even if we remove bits and pieces from the file, as
2400-- long as it isn't a significant amount, it'll remain unique.
2401--
2402-- One case this doesn't cover is if the server generates a random haiku for
2403-- the user.
2404--
2405-- @param body The body of the page.
2406function clean_404(body)
2407 if ( not(body) ) then
2408 return
2409 end
2410
2411 -- Remove anything that looks like time
2412 body = string.gsub(body, '%d?%d:%d%d:%d%d', "")
2413 body = string.gsub(body, '%d%d:%d%d', "")
2414 body = string.gsub(body, 'AM', "")
2415 body = string.gsub(body, 'am', "")
2416 body = string.gsub(body, 'PM', "")
2417 body = string.gsub(body, 'pm', "")
2418
2419 -- Remove anything that looks like a date (this includes 6 and 8 digit numbers)
2420 -- (this is probably unnecessary, but it's getting pretty close to 11:59 right now, so you never know!)
2421 body = string.gsub(body, '%d%d%d%d%d%d%d%d', "") -- 4-digit year (has to go first, because it overlaps 2-digit year)
2422 body = string.gsub(body, '%d%d%d%d%-%d%d%-%d%d', "")
2423 body = string.gsub(body, '%d%d%d%d/%d%d/%d%d', "")
2424 body = string.gsub(body, '%d%d%-%d%d%-%d%d%d%d', "")
2425 body = string.gsub(body, '%d%d%/%d%d%/%d%d%d%d', "")
2426
2427 body = string.gsub(body, '%d%d%d%d%d%d', "") -- 2-digit year
2428 body = string.gsub(body, '%d%d%-%d%d%-%d%d', "")
2429 body = string.gsub(body, '%d%d%/%d%d%/%d%d', "")
2430
2431 -- Remove anything that looks like a path (note: this will get the URI too) (note2: this interferes with the date removal above, so it can't be moved up)
2432 body = string.gsub(body, "/[^ ]+", "") -- Unix - remove everything from a slash till the next space
2433 body = string.gsub(body, "[a-zA-Z]:\\[^ ]+", "") -- Windows - remove everything from a "x:\" pattern till the next space
2434
2435 -- If we have SSL available, save us a lot of memory by hashing the page (if SSL isn't available, this will work fine, but
2436 -- take up more memory). If we're debugging, don't hash (it makes things far harder to debug).
2437 if(have_ssl and nmap.debugging() == 0) then
2438 return openssl.md5(body)
2439 end
2440
2441 return body
2442end
2443
2444---Try requesting a non-existent file to determine how the server responds to
2445-- unknown pages ("404 pages")
2446--
2447-- This tells us
2448-- * what to expect when a non-existent page is requested, and
2449-- * if the server will be impossible to scan.
2450--
2451-- If the server responds with a 404 status code, as it is supposed to, then
2452-- this function simply returns 404. If it contains one of a series of common
2453-- status codes, including unauthorized, moved, and others, it is returned like
2454-- a 404.
2455--
2456-- I (Ron Bowes) have observed one host that responds differently for three
2457-- scenarios:
2458-- * A non-existent page, all lowercase (a login page)
2459-- * A non-existent page, with uppercase (a weird error page that says,
2460-- "Filesystem is corrupt.")
2461-- * A page in a non-existent directory (a login page with different font
2462-- colours)
2463--
2464-- As a result, I've devised three different 404 tests, one to check each of
2465-- these conditions. They all have to match, the tests can proceed; if any of
2466-- them are different, we can't check 404s properly.
2467--
2468-- @param host The host object.
2469-- @param port The port to which we are establishing the connection.
2470-- @return status Did we succeed?
2471-- @return result If status is false, result is an error message. Otherwise,
2472-- it's the code to expect (typically, but not necessarily,
2473-- '404').
2474-- @return body Body is a hash of the cleaned-up body that can be used when
2475-- detecting a 404 page that doesn't return a 404 error code.
2476function identify_404(host, port)
2477 local data
2478 local bad_responses = { 301, 302, 400, 401, 403, 499, 501, 503 }
2479
2480 -- The URLs used to check 404s
2481 local URL_404_1 = '/nmaplowercheck' .. os.time(os.date('*t'))
2482 local URL_404_2 = '/NmapUpperCheck' .. os.time(os.date('*t'))
2483 local URL_404_3 = '/Nmap/folder/check' .. os.time(os.date('*t'))
2484
2485 data = get(host, port, URL_404_1,{redirect_ok=false})
2486 if(data == nil) then
2487 stdnse.debug1("HTTP: Failed while testing for 404 status code")
2488 return false, "Failed while testing for 404 error message"
2489 end
2490
2491 if(data.status and data.status == 404) then
2492 stdnse.debug1("HTTP: Host returns proper 404 result.")
2493 return true, 404
2494 end
2495
2496 if(data.status and data.status == 200) then
2497 stdnse.debug1("HTTP: Host returns 200 instead of 404.")
2498
2499 -- Clean up the body (for example, remove the URI). This makes it easier to validate later
2500 if(data.body) then
2501 -- Obtain a couple more 404 pages to test different conditions
2502 local data2 = get(host, port, URL_404_2)
2503 local data3 = get(host, port, URL_404_3)
2504 if(data2 == nil or data3 == nil) then
2505 stdnse.debug1("HTTP: Failed while testing for extra 404 error messages")
2506 return false, "Failed while testing for extra 404 error messages"
2507 end
2508
2509 -- Check if the return code became something other than 200.
2510 -- Status code: -1 represents unknown.
2511 -- If the status is nil or the string "unknown" we switch to -1.
2512 if(data2.status ~= 200) then
2513 if(type(data2.status) ~= "number") then
2514 data2.status = -1
2515 end
2516 stdnse.debug1("HTTP: HTTP 404 status changed for second request (became %d).", data2.status)
2517 return false, string.format("HTTP 404 status changed for second request (became %d).", data2.status)
2518 end
2519
2520 -- Check if the return code became something other than 200
2521 if(data3.status ~= 200) then
2522 if(type(data3.status) ~= "number") then
2523 data3.status = -1
2524 end
2525 stdnse.debug1("HTTP: HTTP 404 status changed for third request (became %d).", data3.status)
2526 return false, string.format("HTTP 404 status changed for third request (became %d).", data3.status)
2527 end
2528
2529 -- Check if the returned bodies (once cleaned up) matches the first returned body
2530 local clean_body = clean_404(data.body)
2531 local clean_body2 = clean_404(data2.body)
2532 local clean_body3 = clean_404(data3.body)
2533 if(clean_body ~= clean_body2) then
2534 stdnse.debug1("HTTP: Two known 404 pages returned valid and different pages; unable to identify valid response.")
2535 stdnse.debug1("HTTP: If you investigate the server and it's possible to clean up the pages, please post to nmap-dev mailing list.")
2536 return false, "Two known 404 pages returned valid and different pages; unable to identify valid response."
2537 end
2538
2539 if(clean_body ~= clean_body3) then
2540 stdnse.debug1("HTTP: Two known 404 pages returned valid and different pages; unable to identify valid response (happened when checking a folder).")
2541 stdnse.debug1("HTTP: If you investigate the server and it's possible to clean up the pages, please post to nmap-dev mailing list.")
2542 return false, "Two known 404 pages returned valid and different pages; unable to identify valid response (happened when checking a folder)."
2543 end
2544
2545 return true, 200, clean_body
2546 end
2547
2548 stdnse.debug1("HTTP: The 200 response didn't contain a body.")
2549 return true, 200
2550 end
2551
2552 -- Loop through any expected error codes
2553 for _,code in pairs(bad_responses) do
2554 if(data.status and data.status == code) then
2555 stdnse.debug1("HTTP: Host returns %s instead of 404 File Not Found.", get_status_string(data))
2556 return true, code
2557 end
2558 end
2559
2560 stdnse.debug1("Unexpected response returned for 404 check: %s", get_status_string(data))
2561
2562 return true, data.status
2563end
2564
2565--- Determine whether or not the page that was returned is a 404 page.
2566--
2567-- This is actually a pretty simple function, but it's best to keep this logic
2568-- close to <code>identify_404</code>, since they will generally be used
2569-- together.
2570--
2571-- @param data The data returned by the HTTP request
2572-- @param result_404 The status code to expect for non-existent pages. This is
2573-- returned by <code>identify_404</code>.
2574-- @param known_404 The 404 page itself, if <code>result_404</code> is 200. If
2575-- <code>result_404</code> is something else, this parameter
2576-- is ignored and can be set to <code>nil</code>. This is
2577-- returned by <code>identify_404</code>.
2578-- @param page The page being requested (used in error messages).
2579-- @param displayall [optional] If set to true, don't exclude non-404 errors
2580-- (such as 500).
2581-- @return A boolean value: true if the page appears to exist, and false if it
2582-- does not.
2583function page_exists(data, result_404, known_404, page, displayall)
2584 if(data and data.status) then
2585 -- Handle the most complicated case first: the "200 Ok" response
2586 if(data.status == 200) then
2587 if(result_404 == 200) then
2588 -- If the 404 response is also "200", deal with it (check if the body matches)
2589 if(#data.body == 0) then
2590 -- I observed one server that returned a blank string instead of an error, on some occasions
2591 stdnse.debug1("HTTP: Page returned a totally empty body; page likely doesn't exist")
2592 return false
2593 elseif(clean_404(data.body) ~= known_404) then
2594 stdnse.debug1("HTTP: Page returned a body that doesn't match known 404 body, therefore it exists (%s)", page)
2595 return true
2596 else
2597 return false
2598 end
2599 else
2600 -- If 404s return something other than 200, and we got a 200, we're good to go
2601 stdnse.debug1("HTTP: Page was '%s', it exists! (%s)", get_status_string(data), page)
2602 return true
2603 end
2604 else
2605 -- If the result isn't a 200, check if it's a 404 or returns the same code as a 404 returned
2606 if(data.status ~= 404 and data.status ~= result_404) then
2607 -- If this check succeeded, then the page isn't a standard 404 -- it could be a redirect, authentication request, etc. Unless the user
2608 -- asks for everything (with a script argument), only display 401 Authentication Required here.
2609 stdnse.debug1("HTTP: Page didn't match the 404 response (%s) (%s)", get_status_string(data), page)
2610
2611 if(data.status == 401) then -- "Authentication Required"
2612 return true
2613 elseif(displayall) then
2614 return true
2615 end
2616
2617 return false
2618 else
2619 -- Page was a 404, or looked like a 404
2620 return false
2621 end
2622 end
2623 else
2624 stdnse.debug1("HTTP: HTTP request failed (is the host still up?)")
2625 return false
2626 end
2627end
2628
2629---Check if the response variable contains the given text.
2630--
2631-- Response variable could be a return from a http.get, http.post,
2632-- http.pipeline_go, etc. The text can be:
2633-- * Part of a header ('content-type', 'text/html', '200 OK', etc)
2634-- * An entire header ('Content-type: text/html', 'Content-length: 123', etc)
2635-- * Part of the body
2636--
2637-- The search text is treated as a Lua pattern.
2638--
2639--@param response The full response table from a HTTP request.
2640--@param pattern The pattern we're searching for. Don't forget to escape '-',
2641-- for example, 'Content%-type'. The pattern can also contain
2642-- captures, like 'abc(.*)def', which will be returned if
2643-- successful.
2644--@param case_sensitive [optional] Set to <code>true</code> for case-sensitive
2645-- searches. Default: not case sensitive.
2646--@return result True if the string matched, false otherwise
2647--@return matches An array of captures from the match, if any
2648function response_contains(response, pattern, case_sensitive)
2649 local result, _
2650 local m = {}
2651
2652 -- If they're searching for the empty string or nil, it's true
2653 if(pattern == '' or pattern == nil) then
2654 return true
2655 end
2656
2657 -- Create a function that either lowercases everything or doesn't, depending on case sensitivity
2658 local case = function(pattern) return string.lower(pattern or '') end
2659 if(case_sensitive == true) then
2660 case = function(pattern) return (pattern or '') end
2661 end
2662
2663 -- Set the case of the pattern
2664 pattern = case(pattern)
2665
2666 -- Check the status line (eg, 'HTTP/1.1 200 OK')
2667 m = {string.match(case(response['status-line']), pattern)};
2668 if(m and #m > 0) then
2669 return true, m
2670 end
2671
2672 -- Check the headers
2673 for _, header in pairs(response['rawheader']) do
2674 m = {string.match(case(header), pattern)}
2675 if(m and #m > 0) then
2676 return true, m
2677 end
2678 end
2679
2680 -- Check the body
2681 m = {string.match(case(response['body']), pattern)}
2682 if(m and #m > 0) then
2683 return true, m
2684 end
2685
2686 return false
2687end
2688
2689---Take a URI or URL in any form and convert it to its component parts.
2690--
2691-- The URL can optionally have a protocol definition ('http://'), a server
2692-- ('scanme.insecure.org'), a port (':80'), a URI ('/test/file.php'), and a
2693-- query string ('?username=ron&password=turtle'). At the minimum, a path or
2694-- protocol and url are required.
2695--
2696--@param url The incoming URL to parse
2697--@return A table containing the result, which can have the following fields:
2698-- * protocol
2699-- * hostname
2700-- * port
2701-- * uri
2702-- * querystring
2703-- All fields are strings except querystring, which is a table
2704-- containing name=value pairs.
2705function parse_url(url)
2706 local result = {}
2707
2708 -- Save the original URL
2709 result['original'] = url
2710
2711 -- Split the protocol off, if it exists
2712 local colonslashslash = string.find(url, '://')
2713 if(colonslashslash) then
2714 result['protocol'] = string.sub(url, 1, colonslashslash - 1)
2715 url = string.sub(url, colonslashslash + 3)
2716 end
2717
2718 -- Split the host:port from the path
2719 local slash, host_port
2720 slash = string.find(url, '/')
2721 if(slash) then
2722 host_port = string.sub(url, 1, slash - 1)
2723 result['path_query'] = string.sub(url, slash)
2724 else
2725 -- If there's no slash, then it's just a URL (if it has a http://) or a path (if it doesn't)
2726 if(result['protocol']) then
2727 result['host_port'] = url
2728 else
2729 result['path_query'] = url
2730 end
2731 end
2732 if(host_port == '') then
2733 host_port = nil
2734 end
2735
2736 -- Split the host and port apart, if possible
2737 if(host_port) then
2738 local colon = string.find(host_port, ':')
2739 if(colon) then
2740 result['host'] = string.sub(host_port, 1, colon - 1)
2741 result['port'] = tonumber(string.sub(host_port, colon + 1))
2742 else
2743 result['host'] = host_port
2744 end
2745 end
2746
2747 -- Split the path and querystring apart
2748 if(result['path_query']) then
2749 local question = string.find(result['path_query'], '?')
2750 if(question) then
2751 result['path'] = string.sub(result['path_query'], 1, question - 1)
2752 result['raw_querystring'] = string.sub(result['path_query'], question + 1)
2753 else
2754 result['path'] = result['path_query']
2755 end
2756
2757 -- Split up the query, if necessary
2758 if(result['raw_querystring']) then
2759 result['querystring'] = {}
2760 local values = stdnse.strsplit('&', result['raw_querystring'])
2761 for i, v in ipairs(values) do
2762 local name, value = table.unpack(stdnse.strsplit('=', v))
2763 result['querystring'][name] = value
2764 end
2765 end
2766
2767 -- Get the extension of the file, if any, or set that it's a folder
2768 if(string.match(result['path'], "/$")) then
2769 result['is_folder'] = true
2770 else
2771 result['is_folder'] = false
2772 local split_str = stdnse.strsplit('%.', result['path'])
2773 if(split_str and #split_str > 1) then
2774 result['extension'] = split_str[#split_str]
2775 end
2776 end
2777 end
2778
2779 return result
2780end
2781
2782---This function should be called whenever a valid path (a path that doesn't
2783-- contain a known 404 page) is discovered.
2784--
2785-- It will add the path to the registry in several ways, allowing other scripts
2786-- to take advantage of it in interesting ways.
2787--
2788--@param host The host the path was discovered on (not necessarily the host
2789-- being scanned).
2790--@param port The port the path was discovered on (not necessarily the port
2791-- being scanned).
2792--@param path The path discovered. Calling this more than once with the same
2793-- path is okay; it'll update the data as much as possible instead
2794-- of adding a duplicate entry
2795--@param status [optional] The status code (200, 404, 500, etc). This can be
2796-- left off if it isn't known.
2797--@param links_to [optional] A table of paths that this page links to.
2798--@param linked_from [optional] A table of paths that link to this page.
2799--@param contenttype [optional] The content-type value for the path, if it's known.
2800function save_path(host, port, path, status, links_to, linked_from, contenttype)
2801 -- Make sure we have a proper hostname and port
2802 host = stdnse.get_hostname(host)
2803 if(type(port) == 'table') then
2804 port = port.number
2805 end
2806
2807 -- Parse the path
2808 local parsed = parse_url(path)
2809
2810 -- Add to the 'all_pages' key
2811 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'all_pages'}, parsed['path'])
2812
2813 -- Add the URL with querystring to all_pages_full_query
2814 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'all_pages_full_query'}, parsed['path_query'])
2815
2816 -- Add the URL to a key matching the response code
2817 if(status) then
2818 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'status_codes', status}, parsed['path'])
2819 end
2820
2821 -- If it's a directory, add it to the directories list; otherwise, add it to the files list
2822 if(parsed['is_folder']) then
2823 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'directories'}, parsed['path'])
2824 else
2825 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'files'}, parsed['path'])
2826 end
2827
2828
2829 -- If we have an extension, add it to the extensions key
2830 if(parsed['extension']) then
2831 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'extensions', parsed['extension']}, parsed['path'])
2832 end
2833
2834 -- Add an entry for the page and its arguments
2835 if(parsed['querystring']) then
2836 -- Add all scripts with a querystring to the 'cgi' and 'cgi_full_query' keys
2837 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'cgi'}, parsed['path'])
2838 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'cgi_full_query'}, parsed['path_query'])
2839
2840 -- Add the query string alone to the registry (probably not necessary)
2841 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'cgi_querystring', parsed['path'] }, parsed['raw_querystring'])
2842
2843 -- Add the individual arguments for the page, along with their values
2844 for key, value in pairs(parsed['querystring']) do
2845 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'cgi_args', parsed['path']}, parsed['querystring'])
2846 end
2847 end
2848
2849 -- Save the pages it links to
2850 if(links_to) then
2851 if(type(links_to) == 'string') then
2852 links_to = {links_to}
2853 end
2854
2855 for _, v in ipairs(links_to) do
2856 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'links_to', parsed['path_query']}, v)
2857 end
2858 end
2859
2860 -- Save the pages it's linked from (we save these in the 'links_to' key, reversed)
2861 if(linked_from) then
2862 if(type(linked_from) == 'string') then
2863 linked_from = {linked_from}
2864 end
2865
2866 for _, v in ipairs(linked_from) do
2867 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'links_to', v}, parsed['path_query'])
2868 end
2869 end
2870
2871 -- Save it as a content-type, if we have one
2872 if(contenttype) then
2873 stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'content-type', contenttype}, parsed['path_query'])
2874 end
2875end
2876
2877return _ENV;