· 8 years ago · Jan 10, 2018, 05:04 AM
1"""
2Pocket Calibre Recipe v1.4
3"""
4from calibre import strftime
5from calibre.web.feeds.news import BasicNewsRecipe
6from string import Template
7import json
8import operator
9import re
10import tempfile
11import urllib
12import urllib2
13
14
15__license__ = 'GPL v3'
16__copyright__ = '''
172010, Darko Miletic <darko.miletic at gmail.com>
182011, Przemyslaw Kryger <pkryger at gmail.com>
192012-2013, tBunnyMan <Wag That Tail At Me dot com>
20'''
21
22
23class Pocket(BasicNewsRecipe):
24 title = 'Pocket'
25 __author__ = 'Darko Miletic, Przemyslaw Kryger, Keith Callenberg, tBunnyMan'
26 description = '''Personalized news feeds. Go to getpocket.com to setup up
27 your news. This version displays pages of articles from
28 oldest to newest, with max & minimum counts, and marks
29 articles read after downloading.'''
30 publisher = 'getpocket.com'
31 category = 'news, custom'
32
33 # Settings people change
34 oldest_article = 7.0
35 max_articles_per_feed = 50
36 minimum_articles = 10
37 mark_as_read_after_dl = True # Set this to False for testing
38 sort_method = 'newest' # MUST be either 'oldest' or 'newest'
39 # To filter by tag this needs to be a single tag in quotes; IE 'calibre'
40 only_pull_tag = None
41
42 # You don't want to change anything under
43 no_stylesheets = True
44 use_embedded_content = False
45 needs_subscription = True
46 articles_are_obfuscated = True
47 apikey = '19eg0e47pbT32z4793Tf021k99Afl889'
48 index_url = u'https://getpocket.com'
49 read_api_url = index_url + u'/v3/get'
50 modify_api_url = index_url + u'/v3/send'
51 legacy_login_url = index_url + u'/l' # We use this to cheat oAuth
52 articles = []
53
54 def get_browser(self, *args, **kwargs):
55 """
56 We need to pretend to be a recent version of safari for the mac to
57 prevent User-Agent checks Pocket api requires username and password so
58 fail loudly if it's missing from the config.
59 """
60 br = BasicNewsRecipe.get_browser(self,
61 user_agent='Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; \
62 en-us) AppleWebKit/533.19.4 (KHTML, like Gecko) \
63 Version/5.0.3 Safari/533.19.4')
64 if self.username is not None and self.password is not None:
65 br.open(self.legacy_login_url)
66 br.select_form(nr=0)
67 br['feed_id'] = self.username
68 br['password'] = self.password
69 br.submit()
70 else:
71 self.user_error("This Recipe requires authentication")
72 return br
73
74 def get_auth_uri(self):
75 """Quick function to return the authentication part of the url"""
76 uri = ""
77 uri = u'{0}&apikey={1!s}'.format(uri, self.apikey)
78 if self.username is None or self.password is None:
79 self.user_error("Username or password is blank.")
80 else:
81 uri = u'{0}&username={1!s}'.format(uri, self.username)
82 uri = u'{0}&password={1!s}'.format(uri, self.password)
83 return uri
84
85 def get_pull_articles_uri(self):
86 uri = ""
87 uri = u'{0}&state={1}'.format(uri, u'unread')
88 uri = u'{0}&contentType={1}'.format(uri, u'article')
89 uri = u'{0}&sort={1}'.format(uri, self.sort_method)
90 uri = u'{0}&count={1!s}'.format(uri, self.max_articles_per_feed)
91 if self.only_pull_tag is not None:
92 uri = u'{0}&tag={1}'.format(uri, self.only_pull_tag)
93 return uri
94
95 def parse_index(self):
96 pocket_feed = []
97 fetch_url = u"{0}?{1}{2}".format(
98 self.read_api_url,
99 self.get_auth_uri(),
100 self.get_pull_articles_uri()
101 )
102 try:
103 request = urllib2.Request(fetch_url)
104 response = urllib2.urlopen(request)
105 pocket_feed = json.load(response)['list']
106 except urllib2.HTTPError as e:
107 self.log.exception(
108 "Pocket returned an error: {0}".format(e.info()))
109 return []
110 except urllib2.URLError as e:
111 self.log.exception(
112 "Unable to connect to getpocket.com's api: {0}\nurl: {1}".format(e, fetch_url))
113 return []
114
115 if len(pocket_feed) < self.minimum_articles:
116 self.mark_as_read_after_dl = False
117 self.user_error(
118 "Only {0} articles retrieved, minimum_articles not reached".format(len(pocket_feed)))
119
120 for pocket_article in pocket_feed.iteritems():
121 self.articles.append({
122 'item_id': pocket_article[0],
123 'title': pocket_article[1]['resolved_title'],
124 'date': pocket_article[1]['time_updated'],
125 'url': u'{0}/a/read/{1}'.format(self.index_url, pocket_article[0]),
126 'real_url': pocket_article[1]['resolved_url'],
127 'description': pocket_article[1]['excerpt'],
128 'sort': pocket_article[1]['sort_id']
129 })
130 self.articles = sorted(self.articles, key=operator.itemgetter('sort'))
131 return [("My Pocket Articles for {0}".format(strftime('[%I:%M %p]')), self.articles)]
132
133 def get_textview(self, url):
134 """
135 Since Pocket's v3 API they removed access to textview. They also
136 redesigned their page to make it much harder to scrape their textview.
137 We need to pull the article, retrieve the formcheck id, then use it
138 to querty for the json version
139 This function will break when pocket hates us
140 """
141 ajax_url = self.index_url + u'/a/x/getArticle.php'
142 soup = self.index_to_soup(url)
143 fc_tag = soup.find('script', text=re.compile("formCheck"))
144 fc_id = re.search(r"formCheck = \'([\d\w]+)\';", fc_tag).group(1)
145 article_id = url.split("/")[-1]
146 data = urllib.urlencode({'itemId': article_id, 'formCheck': fc_id})
147 try:
148 response = self.browser.open(ajax_url, data)
149 except urllib2.HTTPError as e:
150 self.log.exception("unable to get textview {0}".format(e.info()))
151 raise e
152 return json.load(response)['article']
153
154 def get_obfuscated_article(self, url):
155 """
156 Our get_textview returns parsed json so prettify it to something well
157 parsed by calibre.
158 """
159 article = self.get_textview(url)
160 template = Template('<h1>$title</h1><div class="body">$body</div>')
161 with tempfile.NamedTemporaryFile(delete=False) as tf:
162 tmpbody = article['article']
163 for img in article['images']:
164 imgdiv = '<div id="RIL_IMG_{0}" class="RIL_IMG"></div>'.format(
165 article['images'][img]['image_id'])
166 imgtag = '<img src="{0}" \>'.format(
167 article['images'][img]['src'])
168 tmpbody = tmpbody.replace(imgdiv, imgtag)
169
170 tf.write(template.safe_substitute(
171 title=article['title'],
172 body=tmpbody
173 ))
174 return tf.name
175
176 def mark_as_read(self, mark_list):
177 actions_list = []
178 for article_id in mark_list:
179 actions_list.append({
180 'action': 'archive',
181 'item_id': article_id
182 })
183 mark_read_url = u'{0}?actions={1}{2}'.format(
184 self.modify_api_url,
185 json.dumps(actions_list, separators=(',', ':')),
186 self.get_auth_uri()
187 )
188 try:
189 request = urllib2.Request(mark_read_url)
190 urllib2.urlopen(request)
191 except urllib2.HTTPError as e:
192 self.log.exception(
193 'Pocket returned an error while archiving articles: {0}'.format(e))
194 return []
195 except urllib2.URLError as e:
196 self.log.exception(
197 "Unable to connect to getpocket.com's modify api: {0}".format(e))
198 return []
199
200 def cleanup(self):
201 if self.mark_as_read_after_dl:
202 self.mark_as_read([x['item_id'] for x in self.articles])
203 else:
204 pass
205
206 def default_cover(self, cover_file):
207 """
208 Create a generic cover for recipes that don't have a cover
209 This override adds time to the cover
210 """
211 try:
212 from calibre.ebooks.covers import calibre_cover2
213 title = self.title if isinstance(self.title, unicode) else \
214 self.title.decode('utf-8', 'replace')
215 date = strftime(self.timefmt)
216 time = strftime('[%I:%M %p]')
217 img_data = calibre_cover2(title, date, time)
218 cover_file.write(img_data)
219 cover_file.flush()
220 except:
221 self.log.exception('Failed to generate default cover')
222 return False
223 return True
224
225 def user_error(self, error_message):
226 if hasattr(self, 'abort_recipe_processing'):
227 self.abort_recipe_processing(error_message)
228 else:
229 self.log.exception(error_message)
230 raise RuntimeError(error_message)
231
232# vim:ft=python tabstop=8 expandtab shiftwidth=4 softtabstop=4