· 6 years ago · Nov 19, 2019, 01:58 PM
1#!/usr/bin/env python2
2
3from __future__ import absolute_import, division, print_function
4
5import hashlib
6import io
7import os
8import struct
9import sys
10import time
11import zipfile
12
13try:
14 # Python 3
15 from urllib.request import Request, URLError, urlopen
16except ImportError:
17 # Python 2
18 from urllib2 import Request, URLError, urlopen
19
20# fwfetcher.py - a program to extract the Kinect audio firmware from an Xbox360
21# system update. This program includes substantial portions of extract360.py,
22# which is copyright Rene Ladan and others as noted below and provided under
23# the BSD 2-clause license.
24
25"""Program to extract typical XBox 360 files.
26 It can handle LIVE/PIRS, CON (partially), FMIM, and XUIZ files.
27 What about CRA (aka .arc) files? (Dead Rising demo)
28 Copyright (c) 2007, 2008, Rene Ladan <r.c.ladan@gmail.com>, 2-claused BSD
29 license. Portions from various contributors as mentioned in-source.
30 Note that it dumps UTF-16 characters in text strings as-is.
31"""
32
33
34# Minor compatibility shim
35if sys.version_info[0] < 3:
36 input = raw_input
37 range = xrange
38
39###############################################################################
40
41def check_size(fsize, minsize):
42 """Ensure that the filesize is at least minsize bytes.
43 @param fsize the filesize
44 @param minsize the minimal file size
45 @return fsize >= minsize
46 """
47
48 if fsize < minsize:
49 print("Input file too small: %i instead of at least %i bytes." % (
50 fsize, minsize))
51 return fsize >= minsize
52
53###############################################################################
54
55def nice_open_file(filename):
56 """Checks if the output file with the given name already exists,
57 and if so, asks for overwrite permission.
58 @param filename name of the output file to open
59 @return overwrite permission
60 """
61
62 if os.path.isfile(filename):
63 print(filename, "already exists, overwrite? (y/n)", end=' ')
64 answer = input("")
65 return len(answer) > 0 and answer[0] in ["Y", "y"]
66 else:
67 return True
68
69###############################################################################
70
71def nice_open_dir(dirname):
72 """Checks if the output directory with the given name already exists,
73 and if so, asks for overwrite permission. This means that any file
74 in that directory might be overwritten.
75 @param dirname name of the output directory to open
76 @return overwrite permission
77 """
78
79 if os.path.isdir(dirname):
80 print(dirname, "already exists, ok to overwrite files in it? (y/n)",
81 end=' ')
82 answer = input("")
83 return len(answer) > 0 and answer[0] in ["Y", "y"]
84 else:
85 return True
86
87###############################################################################
88
89def do_mkdir(dirname):
90 """Version of os.mkdir() which does not throw an exception if the directory
91 already exists.
92 @param dirname name of the directory to create
93 """
94
95 try:
96 os.mkdir(dirname)
97 except OSError as e:
98 if e.errno == 17:
99 pass # directory already exists
100
101###############################################################################
102
103def strip_blanks(instring):
104 """Strip the leading and trailing blanks from the input string.
105 Blanks are: 0x00 (only trailing) space \t \n \r \v \f 0xFF
106 @param instring the input string
107 @return stripped version of instring
108 """
109
110 rstr = instring.rstrip(b"\0 \t\n\r\v\f\377")
111 return rstr.lstrip(b" \t\n\r\v\f\377")
112
113###############################################################################
114
115def open_info_file(infile):
116 """Open the informational text file.
117 The name is based on that of the input file.
118 @param infile pointer to the input file
119 @return pointer to the informational text file or None if there was no
120 overwrite permission
121 """
122
123 txtname = os.path.basename(infile.name) + ".txt"
124 if nice_open_file(txtname):
125 print("Writing information file", txtname)
126 txtfile = open(txtname, "w")
127 return txtfile
128 else:
129 return None
130
131###############################################################################
132
133def dump_png(infile, pnglen, maxlen, pngid):
134 """Dump the embedded PNG file from the archive file to an output file.
135 @param infile pointer to the archive file
136 @param pnglen size of the PNG file in bytes
137 @param maxlen maximum size of the PNG file in bytes
138 @param pngid indicates if this is the first or second PNG file.
139 """
140
141 # dump PNG icon
142 if pnglen <= maxlen:
143 outname = os.path.basename(infile.name) + "_" + pngid + ".png"
144 if nice_open_file(outname):
145 buf = infile.read(pnglen)
146 print("Writing PNG file", outname)
147 with open(outname, "wb") as outfile:
148 print(buf, end=' ', file=outfile)
149 else:
150 print("PNG image %s too large (%i instead of maximal %i bytes), "
151 "file not written." % (pngid, pnglen, maxlen))
152
153###############################################################################
154
155def dump_info(infile, txtfile, what):
156 """Dumps the 9 information strings from the input file.
157 @param infile pointer to the input file
158 @param txtfile pointer to the resulting text file
159 @param what indicates if the information consists of titles or
160 descriptions
161 """
162
163 print("\n", what, ":", file=txtfile)
164 for i in range(9):
165 info = strip_blanks(infile.read(0x100))
166 if len(info) > 0:
167 print(lang[i], ":", info, file=txtfile)
168
169###############################################################################
170
171def mstime(intime):
172 """Convert the time given in Microsoft format to a normal time tuple.
173 @param intime the time in Microsoft format
174 @return the time tuple
175 """
176
177 num_d = (intime & 0xFFFF0000) >> 16
178 num_t = intime & 0x0000FFFF
179 # format below is : year, month, day, hour, minute, second,
180 # weekday (Monday), yearday (unused), DST flag (guess)
181 return ((num_d >> 9) + 1980, (num_d >> 5) & 0x0F, num_d & 0x1F,
182 (num_t & 0xFFFF) >> 11, (num_t >> 5) & 0x3F, (num_t & 0x1F) * 2,
183 0, 0, -1)
184
185###############################################################################
186
187def do_utime(targetname, atime, mtime):
188 """Set the access and update date/time of the target.
189 Taken from tarfile.py (builtin lib)
190 @param targetname name of the target
191 @param atime the desired access date/time
192 @param mtime the desired update date/time
193 """
194
195 if not hasattr(os, "utime"):
196 return
197 if not (sys.platform == "win32" and os.path.isdir(targetname)):
198 # Using utime() on directories is not allowed on Win32 according to
199 # msdn.microsoft.com
200 os.utime(targetname,
201 (time.mktime(mstime(atime)), time.mktime(mstime(mtime))))
202
203###############################################################################
204
205def check_sha1(sha1, entry, infile, start, end):
206 """Check the SHA1 value of the specified range of the input file.
207 @param sha1 the reported SHA1 value
208 @param entry the id of the hash
209 @param infile the input file to check
210 @param start the start position
211 @param end the end position
212 @return string reporting if the hash is correct
213 """
214
215 infile.seek(start)
216 found_sha1 = hashlib.sha1(infile.read(end - start))
217 found_digest = found_sha1.digest()
218 # SHA1 hashes are 20 bytes (160 bits) long
219 ret = "SHA1 " + hex(entry) + " "
220 if found_digest == sha1:
221 return ret + "ok (" + found_sha1.hexdigest() + ")"
222 else:
223 hexdig = ""
224 for i in sha1:
225 if ord(i) < 10:
226 val = "0"
227 else:
228 val = ""
229 val += hex(ord(i))[2:]
230 hexdig += val
231 return ret + "wrong (should be " + hexdig + " actual " + \
232 found_sha1.hexdigest() + ")"
233
234###############################################################################
235
236def get_cluster(startclust, offset):
237 """get the real starting cluster"""
238 rst = 0
239 # BEGIN wxPirs
240 while startclust >= 170:
241 startclust //= 170
242 rst += (startclust + 1) * offset
243 # END wxPirs
244 return rst
245
246###############################################################################
247
248def fill_directory(infile, txtfile, contents, firstclust, makedir, start,
249 offset):
250 """Fill the directory structure with the files contained in the archive.
251 @param infile pointer to the archive
252 @param txtfile pointer to the resulting information text file
253 @param contents contains the directory information
254 @param firstclust address of the starting cluster of the first file in
255 infile (in 4kB blocks, minus start bytes)
256 @param makedir flag if directory should be filled, useful if only return
257 is wanted
258 @param start start of directory data
259 @param offset increment for calculating real starting cluster
260 """
261
262 # dictionary which holds the directory structure,
263 # patch 0xFFFF is the 'root' directory.
264 paths = {0xFFFF: ""}
265
266 oldpathind = 0xFFFF # initial path, speed up file/dir creation
267
268 for i in range(0x1000 * firstclust // 64):
269 cur = contents[i * 64:(i + 1) * 64]
270 if ord(cur[40:41]) == 0:
271 # if filename length is zero, we're done
272 break
273 (outname, namelen, clustsize1, val1, clustsize2, val2, startclust,
274 val3) = struct.unpack("<40sBHBHBHB", cur[0:50])
275 # sizes and starting cluster are 24 bits long
276 clustsize1 += val1 << 16
277 clustsize2 += val2 << 16
278 startclust += val3 << 16
279 (pathind, filelen, dati1, dati2) = struct.unpack(">HLLL", cur[50:64])
280
281 if not makedir:
282 continue
283
284 nlen = namelen & ~0xC0
285 if nlen < 1 or nlen > 40:
286 print("Filename length (%i) out of range, skipping file." % nlen)
287 continue
288 outname = outname[0:nlen] # strip trailing 0x00 from filename
289
290 if txtfile is not None:
291 if namelen & 0x80 == 0x80:
292 print("Directory", end=' ', file=txtfile)
293 else:
294 print("File", end=' ', file=txtfile)
295 print("name:", outname, file=txtfile)
296 if namelen & 0x40 == 0x40:
297 print("Bit 6 of namelen is set.", file=txtfile)
298
299 if clustsize1 != clustsize2:
300 print("Cluster sizes don't match (%i != %i), skipping file." % (
301 clustsize1, clustsize2))
302 continue
303 if startclust < 1 and namelen & 0x80 == 0:
304 print("Starting cluster must be 1 or greater, skipping file.")
305 continue
306 if filelen > 0x1000 * clustsize1:
307 print("File length (%i) is greater than the size in clusters "
308 "(%i), skipping file." % (filelen, clustsize1))
309 continue
310
311 if pathind != oldpathind:
312 # working directory changed
313 for _ in range(paths[oldpathind].count("/")):
314 os.chdir("..") # go back to root directory
315 os.chdir(paths[pathind])
316 oldpathind = pathind
317 if namelen & 0x80 == 0x80:
318 # this is a directory
319 paths[i] = paths[pathind] + outname + "/"
320 do_mkdir(outname)
321 else:
322 # this is a file
323 # space between files is set to 0x00
324 adstart = startclust * 0x1000 + start
325 if txtfile is not None:
326 print("Starting: advertized", hex(adstart), file=txtfile)
327
328 # block reading algorithm originally from wxPirs
329 buf = b""
330 while filelen > 0:
331 realstart = adstart + get_cluster(startclust, offset)
332 infile.seek(realstart)
333 buf += infile.read(min(0x1000, filelen))
334 startclust += 1
335 adstart += 0x1000
336 filelen -= 0x1000
337 with open(outname, "wb") as outfile:
338 outfile.write(buf)
339
340 do_utime(outname, dati2, dati1)
341
342 # pop directory
343 for _ in range(paths[oldpathind].count("/")):
344 os.chdir("..")
345
346###############################################################################
347
348def write_common_part(infile, txtfile, png2stop, start):
349 """Writes out the common part of PIRS/LIVE and CON files.
350 @param infile pointer to the PIRS/LIVE or CON file
351 @param txtfile pointer to the resulting text file
352 @param png2stop location where the second PNG image stops
353 (PIRS/LIVE : 0xB000, CON : 0xA000)
354 @param start start of directory data, from wxPirs
355 """
356
357 infile.seek(0x32C)
358 # xbox180 : SHA1 hash of 0x0344-0xB000,
359 # CON : 0x0344 - 0xA000 (i.e. png2stop)
360 mhash = infile.read(20)
361 (mentry_id, content_type) = struct.unpack(">LL", infile.read(8))
362
363 if txtfile is not None:
364 print("\nMaster SHA1 hash :",
365 check_sha1(mhash, mentry_id, infile, 0x0344, png2stop),
366 file=txtfile)
367 print("\nContent type", hex(content_type), ":", end=' ', file=txtfile)
368 # content type list partially from V1kt0R
369 # su20076000_00000000 has type 0x000b0000,
370 # i.e. "Full game demo" & "Theme" ...
371 if content_type == 0:
372 print("(no type)", file=txtfile)
373 elif content_type & 0x00000001:
374 print("Game save", file=txtfile)
375 elif content_type & 0x00000002:
376 print("Game add-on", file=txtfile)
377 elif content_type & 0x00030000:
378 print("Theme", file=txtfile)
379 elif content_type & 0x00090000:
380 print("Video clip", file=txtfile)
381 elif content_type & 0x000C0000:
382 print("Game trailer", file=txtfile)
383 elif content_type & 0x000D0000:
384 print("XBox Live Arcade", file=txtfile)
385 elif content_type & 0x00010000:
386 print("Gamer profile", file=txtfile)
387 elif content_type & 0x00020000:
388 print("Gamer picture", file=txtfile)
389 elif content_type & 0x00040000:
390 print("System update", file=txtfile)
391 elif content_type & 0x00080000:
392 print("Full game demo", file=txtfile)
393 else:
394 print("(unknown)", file=txtfile)
395
396 print("\nDirectory data at (hex)", hex(start), file=txtfile)
397 infile.seek(0x410)
398 dump_info(infile, txtfile, "Titles")
399 dump_info(infile, txtfile, "Descriptions")
400 print("\nPublisher:", strip_blanks(infile.read(0x80)), "\n",
401 file=txtfile)
402 print("\nFilename:", strip_blanks(infile.read(0x80)), "\n",
403 file=txtfile)
404 infile.seek(0x1710)
405 (val1, png1len, png2len) = struct.unpack(">HLL", infile.read(10))
406 if txtfile is not None:
407 print("Value:", val1, file=txtfile)
408
409 if png1len > 0:
410 dump_png(infile, png1len, 0x571A - 0x171A, "1")
411
412 if png2len > 0:
413 infile.seek(0x571A)
414 dump_png(infile, png2len, png2stop - 0x571A, "2")
415
416 # entries are 64 bytes long
417 # unused entries are set to 0x00
418 infile.seek(start + 0x2F)
419 (firstclust, ) = struct.unpack("<H", infile.read(2))
420
421 infile.seek(start)
422 buf = infile.read(0x1000 * firstclust)
423
424 outname = os.path.basename(infile.name) + ".dir"
425 makedir = nice_open_dir(outname)
426 if makedir:
427 print("Creating and filling content directory", outname)
428 do_mkdir(outname)
429 os.chdir(outname)
430 if png2stop == 0xB000 and start == 0xC000:
431 offset = 0x1000
432 else:
433 offset = 0x2000
434 fill_directory(infile, txtfile, buf, firstclust, makedir, start, offset)
435
436 # table of SHA1 hashes of payload
437 if txtfile is not None:
438 print(file=txtfile)
439 infile.seek(png2stop)
440 buf = infile.read(start - png2stop)
441 numempty = 0
442 for i in range(len(buf) // 24):
443 entry = buf[i * 24: i * 24 + 24]
444 if entry.count(b"\0") < 24:
445 if numempty > 0:
446 print("\nEmpty entries:", numempty, file=txtfile)
447 numempty = 0
448 print("Hash (hex):", end=' ', file=txtfile)
449 for j in range(20):
450 print(hex(ord(entry[j:j + 1])), end=' ', file=txtfile)
451 (j, ) = struct.unpack(">L", entry[20:24])
452 print("\nEntry id:", hex(j), file=txtfile)
453 else:
454 numempty += 1
455
456 print("\nTrailing data (hex):", end=' ', file=txtfile)
457 for i in range(len(buf) - (len(buf) % 24), len(buf)):
458 print(hex(ord(buf[i:i + 1])), end=' ', file=txtfile)
459 print(file=txtfile)
460
461 txtfile.close()
462
463###############################################################################
464
465def handle_live_pirs(infile, fsize):
466 """LIVE and PIRS files are archive files.
467 They contain a certificate, payload, SHA1 checksums,
468 2 icons, textual information, and the files themselves.
469 @param infile pointer to the archive file
470 @param fsize size of infile
471 """
472
473 print("Handling LIVE/PIRS file.")
474
475 if not check_size(fsize, 0xD000):
476 return
477
478 txtfile = open_info_file(infile)
479 if txtfile is not None:
480 print("Certificate (hex):", end=' ', file=txtfile)
481 cert = infile.read(0x100)
482 for i in range(len(cert)):
483 print(hex(ord(cert[i:i + 1])), end=' ', file=txtfile)
484
485 print("\n\nData (hex):", end=' ', file=txtfile)
486 data = infile.read(0x228)
487 for i in range(len(data)):
488 print(hex(ord(data[i:i + 1])), end=' ', file=txtfile)
489 print(file=txtfile)
490
491 # BEGIN wxPirs
492 infile.seek(0xC032) # originally 4 bytes at 0xC030
493 (pathind, ) = struct.unpack(">H", infile.read(2))
494 if pathind == 0xFFFF:
495 start = 0xC000
496 else:
497 start = 0xD000
498 # END wxPirs
499 write_common_part(infile, txtfile, 0xB000, start)
500
501###############################################################################
502
503# End of code taken from extract360.py.
504
505def urlopen_timeout_retry(request, attempts = 5):
506 import socket
507
508 last_error = None
509 for attempt in range(attempts):
510 try:
511 return urlopen(request)
512 except URLError as e:
513 if isinstance(e.reason, socket.timeout):
514 print("Timeout! ", e)
515 else: raise
516 last_error = e
517 except socket.timeout as e:
518 print("Timeout! ", e)
519 last_error = e
520 raise last_error
521
522def getFileOrURL(filename, url):
523 # Check if a file named filename exists on disk.
524 # If so, return its contents. If not, download it, save it, and return its
525 # contents.
526 try:
527 with open(filename, 'rb') as f:
528 print("Found", filename, "cached on disk, using local copy")
529 retval = f.read()
530 return retval
531 except IOError:
532 pass
533
534 print("Downloading", filename, "from", url)
535 request = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
536 response = urlopen_timeout_retry(request)
537
538 print("Reading response...")
539 retval = response.read()
540 # Save downloaded file to disk
541 with open(filename, "wb") as f:
542 f.write(retval)
543 print("done, saved to", filename)
544 return retval
545
546def extractPirsFromZip(systemupdate):
547 print("Extracting $SystemUpdate/FFFE07DF00000001 from system update "
548 "file...")
549 updatefile = io.BytesIO(systemupdate)
550 with zipfile.ZipFile(updatefile) as z:
551 # print(z.namelist())
552 pirs = z.open("$SystemUpdate/FFFE07DF00000001").read()
553 print("done.")
554 return pirs
555
556if __name__ == "__main__":
557 target = "audios.bin"
558 if len(sys.argv) == 2:
559 target = sys.argv[1]
560 if not os.path.isfile(target):
561 fw = getFileOrURL("SystemUpdate.zip",
562 "https://www.xbox.com/system-update-usb")
563 pirs = extractPirsFromZip(fw)
564
565 lang = ["English", "Japanese", "German", "French", "Spanish",
566 "Italian", "Korean", "Chinese", "Portuguese"]
567 bio = io.BytesIO(pirs)
568 basename = "FFFE07DF00000001"
569 bio.name = basename
570 pwd = os.getcwd()
571 handle_live_pirs(bio, len(pirs) - 4)
572
573 os.chdir(pwd)
574 print("Moving audios.bin to current folder")
575 os.rename(os.path.join(basename + ".dir", "audios.bin"), target)
576
577 print("Cleaning up")
578 os.unlink(basename + ".txt")
579 for root, dirs, files in os.walk(basename + ".dir"):
580 for name in files:
581 os.remove(os.path.join(root, name))
582 for name in dirs:
583 os.rmdir(os.path.join(root, name))
584 os.rmdir(root)
585 os.unlink("SystemUpdate.zip")
586 print("Done!")
587 else:
588 print("Already have audios.bin")