· 8 years ago · Apr 24, 2018, 07:54 AM
1from __future__ import with_statement
2''' CHM File decoding support '''
3__license__ = 'GPL v3'
4__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>,' \
5 ' and Alex Bramley <a.bramley at gmail.com>.'
6
7import sys, os, re, shutil
8from tempfile import mkdtemp
9from mimetypes import guess_type as guess_mimetype
10from htmlentitydefs import name2codepoint
11from pprint import PrettyPrinter
12
13from BeautifulSoup import BeautifulSoup
14from chm.chm import CHMFile
15from chm.chmlib import (
16 CHM_RESOLVE_SUCCESS, CHM_ENUMERATE_NORMAL,
17 chm_enumerate, chm_retrieve_object,
18)
19
20from calibre.ebooks.lrf import option_parser as lrf_parser
21from calibre.ebooks.metadata import MetaInformation
22from calibre.ebooks.metadata.opf import OPFCreator, Guide
23from calibre.ebooks.metadata.toc import TOC
24from calibre.ebooks.lrf.html.convert_from import process_file as html_process_file
25
26def option_parser():
27 parser = lrf_parser('Usage: %prog [options] mybook.chm')
28 parser.add_option(
29 '-d', '--output-dir', default='.',
30 help=_('Output directory. Defaults to current directory.'))
31 return parser
32
33class CHMError(Exception):
34 pass
35
36class CHMReader(CHMFile):
37 def __init__(self, input):
38 CHMFile.__init__(self)
39 if not self.LoadCHM(input):
40 raise CHMError("Unable to open CHM file '%s'"%(input,))
41 self._contents = None
42 self._playorder = 0
43 self._metadata = False
44 self._extracted = False
45
46 # we'll be creating two new files on top of the extracted stuff from
47 # the CHM -- OPF metadata and NCX table of contents. Let's put them in
48 # the same place as the '.hhc' file, which is the CHM TOC.
49 self.root, ext = os.path.splitext(self.topics.lstrip('/'))
50 self.opf_path = self.root + ".opf"
51 self.ncx_path = self.root + ".ncx"
52
53 def GetMetadata(self, basedir=os.getcwdu()):
54 '''Gets meta-data from the CHM file into an OPFCreator object.
55Takes an optional 'basedir' argument, which is provided to the created
56meta-data objects so that they can work out relative paths.'''
57
58 self.opf = OPFCreator(basedir, self.title)
59 self.opf.title_sort = self._title_sort()
60
61 # now, attempt to grab vaguely standard metadata from the "home" page.
62 home = BeautifulSoup(self.GetFile(self.home))
63 self._get_authors(home)
64 self._get_publisher(home)
65 self._get_isbn(home)
66 self._get_comments(home)
67 self._get_coverpath(home)
68
69 self.opf.create_manifest(map(lambda x: (x, guess_mimetype(x)[0]), self.Contents()))
70 tocsoup = BeautifulSoup(self.GetTopicsTree())
71 self.toc = self._parse_toc(tocsoup.body.ul, basedir)
72 # we are providing an ncx index too, so let's only put top-level
73 # TOC stuff in the spine, for brevity's sake...
74 self.opf.create_spine([item.href for item in self.toc if item.href])
75 self.opf.set_toc(self.toc)
76 self.opf.guide = self._create_guide(tocsoup, basedir)
77 self._metadata = True
78
79 def _title_sort(self):
80 prefixes = ('a ', 'the ')
81 ts = self.title
82 for prefix in prefixes:
83 if ts[0:len(prefix)].lower() == prefix:
84 ts = ts[len(prefix):len(ts)]+", "+ts[0:len(prefix)-1]
85 return ts
86
87 def _metadata_from_table(self, soup, searchfor):
88 td = soup.find('td', text=re.compile(searchfor, flags=re.I))
89 if td is None:
90 return None
91 td = td.parent
92 # there appears to be multiple ways of structuring the metadata
93 # on the home page. cue some nasty special-case hacks...
94 if re.match(r'^\s*'+searchfor+r'\s*$', td.renderContents(), flags=re.I):
95 meta = self._detag(td.findNextSibling('td'))
96 return re.sub('^:', '', meta).strip()
97 else:
98 meta = self._detag(td)
99 return re.sub(r'^[^:]+:', '', meta).strip()
100
101 def _metadata_from_span(self, soup, searchfor):
102 span = soup.find('span', {'class': re.compile(searchfor, flags=re.I)})
103 if span is None:
104 return None
105 # this metadata might need some cleaning up still :/
106 return span.renderContents().strip()
107
108 def _get_authors(self, soup):
109 aut = (self._metadata_from_span(soup, r'author')
110 or self._metadata_from_table(soup, r'^\s*by\s*:?\s+'))
111 if aut is None:
112 self.opf.authors = [u'Unknown']
113 self.opf.author_sort = u''
114 else:
115 aut = re.split(r'\s*(?:,|and)\s*',
116 re.sub(re.compile(r'^\s*by:?\s*', flags=re.I), '', aut))
117 self.opf.authors = aut
118 aut = aut[0].split()
119 # assume sorting by first named author's surname
120 # and further that surname == name.split()[-1]
121 self.opf.author_sort = aut[-1] + ', ' + ' '.join(aut[0:-1])
122
123 def _get_publisher(self, soup):
124 self.opf.publisher = (self._metadata_from_span(soup, 'imprint')
125 or self._metadata_from_table(soup, 'publisher'))
126
127 def _get_isbn(self, soup):
128 isbn = (self._metadata_from_span(soup, 'isbn')
129 or self._metadata_from_table(soup, 'isbn'))
130 self.opf.isbn = re.sub(re.compile(r'^\s*isbn\s*\:', flags=re.I), '', isbn).strip()
131
132 def _get_comments(self, soup):
133 date = (self._metadata_from_span(soup, 'cwdate')
134 or self._metadata_from_table(soup, 'pub date'))
135 pages = (self. _metadata_from_span(soup, 'pages')
136 or self._metadata_from_table(soup, 'pages'))
137 try:
138 # date span can have copyright symbols in it...
139 date = date.replace(u'\u00a9', '').strip()
140 # and pages often comes as '(\d+ pages)'
141 pages = re.search(r'\d+', pages).group(0)
142 self.opf.comments = u'Published %s, %s pages.' % (date, pages)
143 except AttributeError:
144 self.opf.comments = u''
145
146 def _get_coverpath(self, soup):
147 self.opf.cover = None
148 try:
149 self.opf.cover = soup.find('img', alt=re.compile('cover', flags=re.I))['src']
150 except TypeError:
151 # meeehh, no handy alt-tag goodness, try some hackery
152 # the basic idea behind this is that in general, the cover image
153 # has a height:width ratio of ~1.25, whereas most of the nav
154 # buttons are decidedly less than that.
155 # what we do in this is work out that ratio, take 1.25 off it and
156 # save the absolute value when we sort by this value, the smallest
157 # one is most likely to be the cover image, hopefully.
158 r = {}
159 for img in soup('img'):
160 try:
161 r[abs(float(img['height'])/float(img['width'])-1.25)] = img['src']
162 except KeyError:
163 # interestingly, occasionally the only image without height
164 # or width attrs is the cover...
165 r[0] = img['src']
166 l = r.keys()
167 l.sort()
168 self.opf.cover = r[l[0]]
169 # this link comes from the internal html, which is in a subdir
170 if self.opf.cover is not None:
171 self.opf.cover = self.root + "/" + self.opf.cover
172
173 def _create_guide(self, soup, basedir=os.getcwdu()):
174 guide = Guide()
175 guide.set_basedir(basedir)
176 titlepage = Guide.Reference(self.home.lstrip('/'), basedir)
177 titlepage.title = u'About this E-Book'
178 titlepage.type = u'title-page'
179 guide.append(titlepage)
180 # let's try and get useful guide things from our toc soup
181 # map the guide type attribute to name and search regex
182 map = {
183 'toc': [u'Table of Contents', '(?:table of )?contents?'],
184 'copyright-page': [u'Copyright', 'copyright'],
185 'dedication': [u'Dedication', 'dedication'],
186 'preface': [u'Preface', 'preface'],
187 'foreword': [u'Foreword', 'foreword'],
188 'acknowledgements': [u'Acknowledgements', 'acknowledgements'],
189 'bibliography': [u'Bibliography', 'bibliography'],
190 'index': [u'Index', 'index'],
191 'glossary': [u'Glossary', 'glossary'],
192 'colophon': [u'Colophon', 'colophon'],
193 'text': [u'Start of Content', 'chapter 1'],
194 }
195 for type, name in map.items():
196 obj = soup.find('param', {
197 'name': 'Name',
198 'value': re.compile(name[1], re.I)
199 })
200 if obj is None: continue
201 href = obj.parent.find('param', {'name': 'Local'})['value']
202 ref = Guide.Reference(href, basedir)
203 ref.title = name[0]
204 ref.type = type
205 guide.append(ref)
206 return guide
207
208 def _parse_toc(self, ul, basedir=os.getcwdu()):
209 toc = TOC(play_order=self._playorder, base_path=basedir)
210 self._playorder += 1
211 for li in ul('li', recursive=False):
212 href = li.object('param', {'name': 'Local'})[0]['value']
213 if href.count('#'):
214 href, frag = href.split('#')
215 else:
216 frag = None
217 name = self._deentity(li.object('param', {'name': 'Name'})[0]['value'])
218 toc.add_item(href, frag, name, play_order=self._playorder)
219 self._playorder += 1
220 if li.ul:
221 child = self._parse_toc(li.ul)
222 child.parent = toc
223 toc.append(child)
224 return toc
225
226 def _detag(self, tag):
227 str = ""
228 for elem in tag:
229 if hasattr(elem, "contents"):
230 str += self._detag(elem)
231 else:
232 str += self._deentity(elem)
233 return str
234
235 def _deentity(self, elem):
236 def replace_entity(m):
237 if m.group(1)=='#':
238 try:
239 return unichr(int(m.group(2)))
240 except ValueError:
241 return '&#%s;' % m.group(2)
242 try:
243 return unichr(name2codepoint[m.group(2)])
244 except KeyError:
245 return '&%s;' % m.group(2)
246 # rargh nbsp => \xa0, not a real space
247 return re.sub(r'\s+', ' ', re.sub(r'&(#?)([^;]+);', replace_entity, elem).replace(u'\u00a0', ' '))
248
249 def GetFile(self, path):
250 # have to have abs paths for ResolveObject, but Contents() deliberately
251 # makes them relative. So we don't have to worry, re-add the leading /.
252 if path[0] != '/':
253 path = '/' + path
254 res, ui = self.ResolveObject(path)
255 if res != CHM_RESOLVE_SUCCESS:
256 raise CHMError("Unable to locate '%s' within CHM file '%s'"%(path, self.filename))
257 size, data = self.RetrieveObject(ui)
258 if size == 0:
259 raise CHMError("'%s' is zero bytes in length!"%(path,))
260 return data
261
262 def ExtractFiles(self, output_dir=os.getcwdu()):
263 for path in self.Contents():
264 lpath = os.path.join(output_dir, path)
265 self._ensure_dir(lpath)
266 data = self.GetFile(path)
267 with open(lpath, 'wb') as f:
268 if guess_mimetype(path)[0] == ('text/html'):
269 data = self._reformat(data)
270 f.write(data)
271 self._extracted = True
272
273 def _reformat(self, data):
274 try:
275 html = BeautifulSoup(data)
276 except UnicodeEncodeError:
277 # hit some strange encoding problems...
278 print "Unable to parse html for cleaning, leaving it :("
279 return data
280 # nuke javascript...
281 [s.extract() for s in html('script')]
282 # remove forward and back nav bars from the top/bottom of each page
283 # cos they really fuck with the flow of things and generally waste space
284 # since we can't use [a,b] syntax to select arbitrary items from a list
285 # we'll have to do this manually...
286 t = html('table')
287 if t:
288 if (t[0].previousSibling is None
289 or t[0].previousSibling.previousSibling is None):
290 t[0].extract()
291 if (t[-1].nextSibling is None
292 or t[-1].nextSibling.nextSibling is None):
293 t[-1].extract()
294 # for some very odd reason each page's content appears to be in a table
295 # too. and this table has sub-tables for random asides... grr.
296
297 # some images seem to be broken in some chm's :/
298 for img in html('img'):
299 try:
300 # some are supposedly "relative"... lies.
301 while img['src'].startswith('../'): img['src'] = img['src'][3:]
302 # some have ";<junk>" at the end.
303 img['src'] = img['src'].split(';')[0]
304 except KeyError:
305 # and some don't even have a src= ?!
306 pass
307 # now give back some pretty html.
308 return html.prettify()
309
310 def Contents(self):
311 if self._contents is not None:
312 return self._contents
313 paths = []
314 def get_paths(chm, ui, ctx):
315 # skip directories
316 if ui.path[-1] != '/':
317 # and make paths relative
318 paths.append(ui.path.lstrip('/'))
319 chm_enumerate(self.file, CHM_ENUMERATE_NORMAL, get_paths, None)
320 self._contents = paths
321 return self._contents
322
323 def _ensure_dir(self, path):
324 dir = os.path.dirname(path)
325 if not os.path.isdir(dir):
326 os.makedirs(dir)
327
328 def CreateMetafiles(self, output_dir=os.getcwdu()):
329 if not self._metadata:
330 self.GetMetadata(basedir=output_dir)
331 self._ensure_dir(output_dir)
332 opf_fd = open(os.path.join(output_dir, self.opf_path), 'wb')
333 ncx_fd = open(os.path.join(output_dir, self.ncx_path), 'wb')
334 self.opf.render(opf_fd, ncx_fd, self.ncx_path)
335 opf_fd.close()
336 ncx_fd.close()
337
338 def extract_content(self, output_dir=os.getcwdu()):
339 self.ExtractFiles(output_dir=output_dir)
340 self.CreateMetafiles(output_dir=output_dir)
341
342def process_file(f, options, logger):
343 tdir = mkdtemp(prefix='chm2oeb_')
344 f = os.path.abspath(os.path.expanduser(f))
345 if not options.output:
346 ext = '.lrs' if options.lrs else '.lrf'
347 options.output = os.path.splitext(f)[0] + ext
348 rdr = CHMReader(f)
349 print "Extracting CHM to ", tdir
350 rdr.extract_content(tdir)
351 options.opf = os.path.join(tdir, rdr.opf_path)
352 try:
353 html_process_file(os.path.join(tdir, rdr.home.lstrip('/')), options)
354 finally:
355 try:
356 shutil.rmtree(tdir)
357 except:
358 print "Failed to delete tempdir ", tdir
359
360
361def main(args=sys.argv, logger=None):
362 parser = option_parser()
363 options, args = parser.parse_args(args)
364 if len(args) != 2:
365 parser.print_help()
366 print
367 print "FAIL: provide a CHM file as an argument!"
368 return 1
369 process_file(args[1], options, logger)
370# tdir = mkdtemp(prefix='chm2oeb_', dir='.')
371# rdr = CHMReader(args[1])
372# rdr.extract_content(tdir)
373 return 0
374
375if __name__ == '__main__':
376 sys.exit(main())