· 5 years ago · Mar 25, 2020, 08:08 PM
1#!/usr/bin/env python3
2# qposts_research.py
3# version 0.0.1c (20200206)
4# prev ver: https://pastebin.com/79JKLieB
5
6# Below is a set of Python methods to facilitate research/analysis of Qposts.
7# - needs Python (version 3.4+ );
8# - uses only standard Python libraries;
9# - tested only with Python 3.7.5 on Ubuntu 19.10
10# - dictionary of Qpost abbreviations included.
11#
12# To run the script, type "python3 qposts_research.py" in your terminal (at the correct folder-location).
13# With this Python script you can:
14# - Download all Qposts as a single JSON-file from qanon.pub.
15# - Define arbitrary subsets of Qposts and print them out in any order.
16# - Perform a complex Text or Date Search using "and, or, and not"-operators.
17# - Perform a Regular Expression Search or group matching of Qpost data.
18# - Find Qclock-aligned dates, together with the Qposts posted on those dates.
19# - Download images linked in Qposts (100% success rate), including images from references (93% success).
20# - Export your Qposts JSON-file to an SQLite3 Database and perform SQL SELECT-queries from the terminal.
21#
22# Relevant Quotes from Q:
23# Q59: "Combine all posts and analyze."
24# Q993: "Learn how to archive offline."
25# Q22: "Study and prepare."
26#
27# This is Open-Source Freeware/QAnonware;
28# Released As-Is; No Warranty; Use at Own Risk.
29# May God Bless Q-team & QAnons Worldwide.
30# WWG1WGA
31
32import os
33import re
34import json
35import html
36import sqlite3
37import argparse
38import datetime
39import dateutil.parser as dateparser
40import xml.etree.ElementTree as xml_tree
41from urllib.request import Request, urlopen
42from urllib.error import URLError
43from collections import Counter
44from textwrap import wrap
45
46# maximum wrap width for the Qposts "Text" field.
47_MAX_WRAP = 120
48# indented space for displaying nested references.
49_LVL_INDENT = ' ' * 4
50# Determines the initial subset of Qmap IDs (can be 'all', 'first N', 'last N', etc.).
51_INITIAL_SUBSET = '*-1'
52# True = Show Tooltips for known abbreviations in the Qposts "Text" field.
53_SHOW_ABBR_TOOLTIPS = True
54# True = Qclock Hourhand Round-off to Nearest Minute; False = Round to Floor.
55_QCLOCK_ROUND_NEAREST = True
56# Determines the initial visibility of each menu group (0=hidden; 1=visible).
57_VISIBLE_MENU_GROUPS = {'A':1,'B':1,'C':1,'D':1}
58# (Group D is only active if the user has an SQLite3 Qposts.db at _URL_QPOSTS_DB.)
59
60_DT_FORMAT = '%a %d %b %y' # strftime format to display a short date string.
61_DT_FORMAT_L = '%A %d %B %Y' # strftime format to display a long date string.
62_DTM_FORMAT = '%A %d %B %Y %X %Z' # strftime format to display a long date&time string.
63
64ok = '\033[1;48;2;190;190;190;38;2;10;100;20m' # green color for Success.
65er = '\033[1;48;2;190;190;190;38;2;160;30;50m' # red color for Errors.
66mc = '\033[0;48;2;190;190;190;38;2;30;100;50m' # color for input Labels.
67bl = '\033[1;48;2;132;173;191;30m' # background light blue for Qclock dates with at least 1 Qpost.
68mf = '\033[0;38;2;150;190;10m' # SQLite DB link color.
69gr = '\033[38;2;180;180;180m' # color for greyed-out text.
70bw = '\033[1;37;40m' # color for values: Bold white on black.
71tc = '\033[0;33m' # yellow for Qpost Labels + program text.
72iv = '\033[7m' # invert
73cu = '\033[3m' # italic
74bd = '\033[1m' # bold
75ts = '\033[0m' # reset to normal
76
77_SAFE_URL_QPOSTS = '~/Downloads/Qposts.json' # location for the downloaded Qposts.json file.
78_SAFE_URL_QPOSTS_DB = '~/Downloads/Qposts.db' # location for the Qposts SQLite-database.
79_SAFE_URL_QPOSTS_IMAGES = '~/Downloads/Qposts_images/' # location for downloaded Qpost images.
80
81# un-anonymized versions of the url paths above.
82_URL_QPOSTS = os.path.expanduser( _SAFE_URL_QPOSTS )
83_URL_QPOSTS_DB = os.path.expanduser( _SAFE_URL_QPOSTS_DB )
84_URL_QPOSTS_IMAGES = os.path.expanduser( _SAFE_URL_QPOSTS_IMAGES )
85
86_QPOSTS = []
87_QMAP_IDS = []
88_QPOST_KEYS = [ 'timestamp', 'text', 'media', 'references', 'name', 'trip', 'userId', 'link',
89 'source', 'threadId', 'id', 'title', 'subject', 'email', 'timestampDeletion' ]
90# key timestampDeletion: used in 5 Qposts: [124,229,231,232,240] (all on 4plebs).
91# title: used in 248 Qposts (on 4plebs/8chan_cbts); subject: used in all other Qposts (on 8ch/8kun).
92
93_QPOST_DB_COLS = [ 'qmap_id' ]
94_QPOST_DB_COLS.extend( _QPOST_KEYS )
95_QPOST_DB_COLS.__setitem__( 4, 'refs' ) # NB. 'references' is a reserved word in SQLite3.
96
97_DOC_STRFTIME = 'https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior'
98_DOC_SQLITE_FUNC = 'https://www.sqlite.org/lang_corefunc.html'
99_DOC_SQLITE_SLCT = 'https://www.sqlitetutorial.net/sqlite-select/'
100
101
102# Dictionary of Qpost Abbreviations (Non-exhaustive; Collected from various sources):
103# NB. this dictionary needs careful ordering of the items:
104# - Keys that occur entirely inside another item's Value, must be placed BEFORE all such items (else the link formatting gets mixed up).
105# - Keys that occur entirely inside another item's Key, must be placed AFTER all such items (else only the smaller Key will match).
106# In this way the smaller Keys will have their own tooltip inside the larger Key, and the larger Key will also have its own tooltip,
107# unless the larger Key consists entirely of smaller Keys...
108_QPOST_ABBR = {
109 'AUS': 'Australia',
110 'AZ': 'Arizona',
111 'CA': 'California',
112 'EU': 'European Union',
113 'FL': 'Florida',
114 'FR': 'France',
115 'GER': 'Germany',
116 'HI': 'Hawaii',
117 'HK': 'Hong Kong',
118 'HKG': 'Hong Kong',
119 'LV': 'Las Vegas',
120 'MX': 'Mexico',
121 'NK': 'North Korea',
122 'NY': 'New York',
123 'NYC': 'New York City',
124 'NZ': 'New Zealand',
125 'PAK': 'Pakistan',
126 'RUS': 'Russia',
127 'SA': 'Saudi Arabia',
128 'SK': 'South Korea',
129 'TN': 'Tennessee',
130 'TX': 'Texas',
131 'UK': 'United Kingdom',
132 'UN': 'United Nations',
133 'US': 'United States',
134 'USA': 'United States of America',
135 'UT': 'Utah',
136 'VA': 'Virginia',
137 'WASH': 'Washington',
138 'ATL': 'Atlanta Airport',
139 'IAD': 'Dulles International Airport (Washington)',
140 'PVG': 'Shanghai Pudong Airport',
141 'AF1': 'Air Force 1 (Presidential Airplane)',
142 'AG': 'Attorney General',
143 'AMB': 'Ambassador',
144 'CEO': 'Chief Executive Officer',
145 'CEOs': 'Chief Executive Officers',
146 'CIGIE': 'Council of Inspectors General on Integrity and Efficiency',
147 'DAG': 'Deputy Attorney General',
148 'IG': 'Inspector General',
149 'NY_AG': 'Attorney General of New York State',
150 'POTUS': 'President of the United States',
151 'SCJ': 'Supreme Court Justice',
152 'SD': 'State Department',
153 'SIG': 'Signal; Special Interest Group',
154 'SIT RM': 'Situation Room (White House)',
155 'VP': 'Vice President',
156 'WH': 'White House',
157 'DoD': 'Department of Defense',
158 'DoT': 'Department of Transportation',
159 'DOJ': 'Department Of Justice',
160 'DOE': 'Department Of Energy',
161 'DHS': 'Department of Homeland Security',
162 'DNC': 'Democratic National Committee',
163 'A\'s': 'Agencies',
164 'ABCs': 'Alphabet agencies (Acronyms of US governmental agencies)',
165 'AFIA': 'Air Force Intelligence Agency',
166 'AIA': 'Army Intelligence Agency',
167 'CIA': 'Central Intelligence Agency',
168 'C_A': 'CIA (Lacking Intelligence)',
169 'DARPA': 'Defense Advanced Research Projects Agency',
170 'DIA': 'Defense Intelligence Agency',
171 'DNI': 'Director of National Intelligence',
172 'ESC': 'Electronic Security Command (USAF)',
173 'FBI': 'Federal Bureau of Investigation',
174 'FISA': 'Foreign Intelligence Surveillance Act',
175 'FISC': 'Foreign Intelligence Surveillance Court (FISA Court)',
176 'FOIA': 'Freedom Of Information Act',
177 'ICE': 'U.S. Immigration and Customs Enforcement',
178 'INSCOM': 'United States Army Intelligence and Security Command',
179 'IRS': 'Internal Revenue Agency',
180 'ITAC': 'Intelligence and Threat Analysis Center (US Army)',
181 'NASA': 'National Aeronautics and Space Administration',
182 'NCTC': 'National Counter Terrorism Center',
183 'NG': 'National Guard',
184 'NOIC': 'Naval Operational Intelligence Center (US Navy)',
185 'NPIC': 'National Photographic Intelligence Center (CIA)',
186 'NSA': 'National Security Agency',
187 'OCMC': 'Overhead Collection Management Center (NSA)',
188 'OIG': 'Office of the Inspector General',
189 'TSA': 'Transportation Security Administration',
190 'SRD': 'Special Research Detachment (US Army)',
191 'SS': 'Secret Service',
192 'USSS': 'United States Secret Service',
193 'FVEY': 'Five Eyes: an intelligence alliance comprising USA, UK, CAN, AUS, NZ',
194 '5 Eyes': 'FVEY (an intelligence alliance comprising USA, UK, CAN, AUS, NZ)',
195 'GCHQ Bude': 'UK Government satellite ground station and eavesdropping centre',
196 'GCHQ': 'Government Communications Headquarters (UK)',
197 'MI5': 'Military Intelligence Section 5 (UK Security Service)',
198 'MI6': 'Military Intelligence Section 6 (UK Secret Intelligence Service)',
199 'SIS': 'UK Secret Intelligence Service',
200 'MOSSAD': 'Israeli Secret Intelligence Service',
201 'MOS': 'MOSSAD (Israeli Secret Intelligence Service)',
202 'MSM': 'Mainstream Media',
203 'ARM': 'Anti-Republican Media',
204 'AP': 'Associated Press',
205 'AURN': 'American Urban Radio Networks',
206 'ABC': 'American Broadcasting Company; Alphabet agencies',
207 'BBC': 'British Broadcasting Corporation',
208 'BUZZF': 'BuzzFeed',
209 'CBS': 'Columbia Broadcasting System',
210 'CNN': 'Cable News Network',
211 'CNBC': 'Consumer News and Business Channel',
212 'HuffPo': 'Huffington Post',
213 'LAT': 'Los Angeles Times',
214 'NBC': 'National Broadcasting Company',
215 'MSNBC': 'US TV network partnership between Microsoft and NBC',
216 'NPR': 'National Public Radio',
217 'NYT': 'New York Times',
218 'OANN': 'One America News Network',
219 'PBS': 'Public Broadcasting Service',
220 'WaPo': 'Washington Post',
221 'WAPO': 'Washington Post',
222 'WASHPOST': 'Washington Post',
223 'WSJ': 'Wall Street Journal',
224 'FB': 'Facebook',
225 'GOOG': 'Google',
226 'WL': 'WikiLeaks',
227 'YT': 'YouTube',
228 'JFK JR': 'John F. Kennedy Junior (son of President John F. Kennedy)',
229 'JFK': 'John Fitzgerald Kennedy (35th US President); Gen. John Francis Kelly',
230 'DJT': 'Donald John Trump (45th US President)',
231 'Flynn JR': 'Michael Flynn Junior (son of General Flynn)',
232 'GHWB': 'George Herbert Walker Bush (41st US President)',
233 'GWB': 'George W. Bush (43rd US President)',
234 'HRC': 'Hillary Rodham Clinton (Secretary of State in Obama\'s first term)',
235 'Huma': 'Huma Abedin (Personal Assistant to Hillary Clinton)',
236 'IA': 'Information Assurance',
237 'IC': 'Intelligence Community',
238 'SC': 'Supreme Court; Special Counsel; Sara Carter (investigative reporter)',
239 'Perkins Coie': 'DNC’s private law firm',
240 'Fusion GPS': 'phony-intel firm was paid $1,024,408 by HRC/Perkins-Coie for creating the Steel Dossier',
241 'Crowdstrike': 'CA-based Cyber-security company that falsely claimed that Russia had hacked the DNC servers',
242 '\n+++': 'House of Saud', # added \n for best result.
243 '\n++': 'Rothschild family',
244 '\n+': 'George Soros (Globalist billionaire)',
245 '[]': 'kill box (area of interest)',
246 '[R]': 'Renegade (Secret Service codename for Barack Obama); Rothschild (?)',
247 '[E]': 'Eagle (Secret Service codename for Bill Clinton)',
248 '[D]': 'Democrat; Democratic',
249 '[D]s': 'Democrats',
250 '1-800-273-8255': 'Phone number of the Veterans Crisis Line',
251 '11.11.18.': 'IP address range of US DoD Network Information Center',
252 '187': 'Police Code for homicide',
253 '212-397-2255': 'Phone number of the Clinton Global Initiative',
254 '302': 'FD-302 form used by the FBI for taking notes during an interview',
255 '404': 'HTTP response code indicating "Page Not Found"',
256 '4-10-20': 'letter values of initials DJT (Donald John Trump)',
257 '4,10,20': 'letter values of initials DJT (Donald John Trump)',
258 '4, 10, 20': 'letter values of initials DJT (Donald John Trump)',
259 '4ch': '4chan (message board previously used by Q)',
260 '5:5': 'Loud and Clear',
261 '7th Floor': '"Shadow Government" within the SD that regularly met on the 7th floor of the Harry S. Truman Building in DC',
262 '8ch': '8chan (message board previously used by Q)',
263 '@JACK': 'Jack Dorsey (CEO of Twitter)',
264 '@Jack': 'Jack Dorsey (CEO of Twitter)',
265 '@Snowden': 'Edward Snowden (CIA/NSA spy who leaked NSA documents)',
266 '@SNOWDEN': 'Edward Snowden (CIA/NSA spy who leaked NSA documents)',
267 'A Cooper': 'Anderson Cooper (CNN anchor, son of Gloria Vanderbilt)',
268 'ADM R': 'Admiral Michael S. Rogers (Director of the NSA)',
269 'Adm R': 'Admiral Michael S. Rogers (Director of the NSA)',
270 'AJ': 'Alex Jones (Radio show host linked to Mossad)',
271 'AL-Q': 'Al-Qaeda (Islamic terrorist group pursuing NATO\'s geostrategic goals)',
272 'AL': 'Al Franken (Sen. D-MN)',
273 'AM': 'Andrew McCabe (FBI Deputy Director); Ante Meridiem',
274 'Anon': 'anonymous person',
275 'ANTIFA': '"Anti-Fascists" (Soros backed fascists/domestic terrorists)',
276 'AS': 'Adam Schiff (Rep. D-CA); Antonin Scalia (Supreme Court Associate Justice)',
277 'ASF': 'American Special Forces (?); Administrative Support Facility (?)',
278 'AW': 'Anthony Weiner (convicted pedophile ex-husband of Huma Abedin)',
279 'AWAN': 'Imran Awan (DNC IT staffer who blackmailed House Members)',
280 'B2': 'Stealth bomber; Bill Barr (US Attorney General under Trump)',
281 'B/H C': 'Bill & Hillary Clinton',
282 'BARR': 'Bill Barr (US Attorney General under GHWB and Trump)',
283 'BB': 'Bill Barr (US Attorney General under GHWB and Trump)',
284 'BC': 'Bill Clinton (42nd US President)',
285 'BDT': 'Bulk Data Transfer; Blunt & Direct Time; Bangladeshi Taka (currency)',
286 'BGA': 'Bundesverband Großhandel, Außenhandel (German trade association)',
287 'BHO': 'Barack Hussein Obama (44th US President)',
288 'BIDEN': 'Joseph Biden (VP under Obama)',
289 'BO': 'Board Owner; Barack Obama; Bruce Ohr (Associate Deputy AG)',
290 'BOD': 'Board of Directors',
291 'BODs': 'Boards of Directors',
292 'BP': 'Border Patrol; Bill Priestap (FBI Dep. Dir. of Counterintelligence under Obama and Trump)',
293 'BRENNAN': 'John Brennan (23rd CIA Director)',
294 'BS': 'Bernie Sanders (Sen. I-VT)',
295 'CC': 'Chelsea Clinton (daughter of Bill & Hillary Clinton)',
296 'CF': 'Clinton Foundation',
297 'CFR': 'Council on Foreign Relations',
298 'CHAI': 'Clinton Health Access Initiative',
299 'C-Info': 'Confidential Information',
300 'CLAPPER': 'Director of National Intelligence under Obama',
301 'CLAS': 'Classification; Classified',
302 'CLAS_OP_IAD_': 'Classified Operation at Dulles International Airport (?)',
303 'Clowns In America': 'CIA',
304 'CLOWNS IN AMERICA': 'CIA',
305 'CM': 'CodeMonkey (8kun Admin); Cheryl Mills (Adviser to Hillary Clinton)',
306 'CoC': 'Chain of Command; Chain of Custody',
307 'COC': 'Chain Of Command; Chain Of Custody',
308 'COMEY': 'James Comey (7th FBI Director)',
309 'CORSI': 'Jerome Corsi, Mossad asset/agent',
310 'CoS': 'Chief of Staff',
311 'COV': 'Covert',
312 'COVFEFE': 'Communications Over Various Feeds Electronically For Engagement Act',
313 'CRUZ': 'Ted Cruz (Sen. R-TX)',
314 'CS': 'Chuck Schumer (Sen. D-NY); Christopher Steele (former MI6); Civil Service',
315 'D\'s': 'Democrats',
316 'D’s': 'Democrats',
317 'D+R+I': 'Democrat + Republican + Independent',
318 'D5': 'Highest avalanche rating; December 5th; Chess move; 45=Trump',
319 'DACA': 'Deferred Action for Childhood Arrivals (US immigration policy)',
320 'DC': 'District of Columbia (Washington); Dan Coats (DNI under Trump); Dick Cheney (VP under G.W.Bush)',
321 'DDoS': 'Directed Denial of Service (computer attack)',
322 'DECLAS': 'Declassification; Declassified',
323 'DEFCON': 'Defense Condition; Definitely Confirmed',
324 'DF': 'Dianne Feinstein (Sen. D-CA)',
325 'DL': 'Driver\'s License; David Laufman (Federal prosecutor); David Lawrence (Counsel to the Assistant AG)',
326 'DM': 'Denis McDonough (White House Chief of Staff under Obama)',
327 'DOA': 'Date Of Arrival; Dead Or Alive',
328 'Donna': 'Donna Brazille (Hillary staffer)',
329 'Dopey': 'Prince Al-Waleed bin Talal bin Abdulaziz al Saud',
330 'DS': 'Deep State',
331 'DWS': 'Debbie Wasserman Schulz (Rep. D-FL, DNC Chair under Obama)',
332 'Eagle': 'Secret Service codename for President Bill Clinton',
333 'Evergreen': 'Secret Service codename for Hillary Clinton',
334 'EBS': 'Emergency Broadcast System',
335 'EC': 'Eric Ciaramella (CIA agent)',
336 'EG': 'Evergreen (Secret Service codename for Hillary Clinton)',
337 'EH': 'Eric Holder (US Attorney General under Obama)',
338 'EM': 'Emergency; Elon Musk (CEO of SpaceX and Tesla Inc.)',
339 'EMP': 'Electromagnetic pulse',
340 'EMS': 'Emergency Medical Services; Emergency Medical System',
341 'EO': 'Executive Order',
342 'EOs': 'Executive Orders',
343 'EPSTEIN': 'Jeffrey Epstein (Billionaire who operated an elite pedophile ring for the Mossad)',
344 'ES': 'Eric Schmidt (CEO of Google); Edward Snowden (CIA double agent who leaked NSA secrets)',
345 'EST': 'Eastern Standard Time',
346 'F + D': 'Foreign and Domestic',
347 'F&F': 'Fast and Furious - Feinstein\'s failed gun sale attempt',
348 'F2F': 'Face to Face',
349 'f2f': 'face to face',
350 'F9': 'Message Authentication Code integrity algorithm used by Facebook',
351 'FED': 'Federal Reserve System (US Central Bank); Federal',
352 'FEINSTEIN': 'Dianne Feinstein (Sen. D-CA)',
353 'FF': 'False Flag',
354 'FG&C': 'For God And Country',
355 'FIRE & FURY': 'President Trump\'s warning to North Korea (8 Aug 2017)',
356 'FISA_T_SURV': 'Targeted Surveillance authorized under Section 702 of the FISA Amendments Act',
357 'FLYNN': 'Gen. Michael T. Flynn (National Security Advisor under Obama, fired for patriotism)',
358 'FY': 'Fiscal Year',
359 'G v E': 'Good versus Evil',
360 'GA': 'Great Awakening',
361 'GANG OF 8': 'Oversight board of the U.S. intelligence community',
362 'GANG OF EIGHT': 'Oversight board of the U.S. intelligence community',
363 'GDP': 'Gross Domestic Product',
364 'GINA': 'Gina Haspel (25th CIA Director)',
365 'GJ': 'Grand Jury',
366 'GOODLATTE': 'Bob Goodlatte (Rep. R-VA)',
367 'GOP': 'Grand Old Party (Republican Party)',
368 'gov\'t': 'Government',
369 'govt': 'Government',
370 'Gov’t': 'Government',
371 'Gov': 'Governor; Government',
372 'GOV': 'Government',
373 'GOWDY': 'Trey Gowdy (Rep. D-SC)',
374 'GPS': 'Global Positioning System',
375 'GRASSLEY': 'Chuck Grassley (Sen. R-IA)',
376 'GS': 'George Soros (Billionaire globalist investor)',
377 'GZ': 'Ground Zero',
378 'HA': 'Huma Abedin (Personal Assistant to Hillary Clinton)',
379 'HAM radio': 'Amateur radio',
380 'HEC': 'House Ethics Committee',
381 'HLS': 'Harvard Law School',
382 'HOLDER': 'Eric Holder (US Attorney General under Obama)',
383 'HOROWITZ': 'Michael Horowitz (DOJ Inspector General)',
384 'HS': 'Homeland Security',
385 'H-relief': 'Haiti earthquake relief effort coordinated by Bill Clinton',
386 'HUBER': 'John Huber (US Attorney for Utah)',
387 'HUMA': 'Harvard University Muslim Alumni; Huma Abedin',
388 'HUMINT': 'Human Intelligence',
389 'HUNTER': 'Hunter Biden (son of Joe Biden)',
390 'HUSSEIN': 'Barack Hussein Obama (44th US President)',
391 'HW': 'Hollywood',
392 'HWOOD': 'Hollywood',
393 'H wood': 'Hollywood',
394 'H-wood': 'Hollywood',
395 'ICBM': 'Inter-Continental Ballistic Missile',
396 'ICIG': 'Inspector General of the Intelligence Community',
397 'ISIS': 'Israeli Secret Intelligence Service; Islamic State in Iraq and Syria',
398 'IW': 'Information Warfare',
399 'IQT': 'In-Q-Tel (Private firm providing information technology to the CIA)',
400 'James 8. Corney': 'Deliberate misspelling of "James B. Comey"',
401 'JA': 'Julian Assange (Founder of Wikileaks)',
402 'JB': 'John Brennan (CIA Director); Joe Biden (VP under Obama); Jim Baker (FBI General Counsel); Jeff Bezos (CEO of Amazon)',
403 'JC': 'James Comey (FBI Director); James Clapper (DNI under Obama); John Carlin (Assistant AG); Josh Campbell (FBI Special Agent)',
404 'JD': 'Jack Dorsey (CEO of Twitter)',
405 'JK': 'John Kerry (Secretary of State under Obama), Jared Kushner (Senior Adviser under Trump)',
406 'JL': 'John Legend (American singer/songwriter)',
407 'John M': 'John McCain (Sen. R-AZ)',
408 'JP': 'John Podesta (WH Chief of Staff under Clinton, Counselor under Obama)',
409 'JR': 'Junior; Jim Rybicki (FBI Chief of Staff under Comey, fired by Wray)',
410 'JS': 'Jeff Sessions (US Attorney General under Trump); John Solomon (investigative reporter)',
411 'Judge K': 'Brett Kavanaugh (Supreme Court Associate Justice)',
412 'Justice K': 'Brett Kavanaugh (Supreme Court Associate Justice)',
413 'KAV': 'Brett Kavanaugh (Supreme Court Associate Justice)',
414 'KC': 'Kevin Clinesmith (FBI attorney)',
415 'KKK': 'Klu Klux Klan (created by the Democrats)',
416 'KM': 'Kelly Magsamen (Special Assistant to the President)',
417 'LARP': 'Live Action Role Player',
418 'LifeLog': 'Pentagon DARPA mass-surveillance project rebranded as Facebook',
419 'LdR': '(Lady) Lynn Forester de Rothschild (married to Evelyn de Rothschild)',
420 'LDR': '(Lady) Lynn Forester de Rothschild',
421 'LL': 'Loretta Lynch (US Attorney General under Obama)',
422 'LLC': 'Limited Liability Company',
423 'LP': 'Lisa Page (FBI Special Counsel)',
424 'LYNCH': 'Loretta Lynch (US Attorney General under Obama)',
425 'M’s': 'Marines',
426 'Maxine W': 'Maxine Waters (Rep. D-CA)',
427 'Mc_I': 'John McCain Institute',
428 'MACRON': 'Emmanuel Macron (President of France)',
429 'MAGA': 'Make America Great Again',
430 'MAY': 'Theresa May (Prime Minster of UK)',
431 'MB': 'Muslim Brotherhood',
432 'MCCABE': 'Andrew McCabe (FBI Deputy Director under Comey, fired)',
433 'MERKEL': 'Angela Merkel (Chancellor of Germany)',
434 'MI': 'Military Intelligence',
435 'MIL': 'Military',
436 'MK': 'Mike Kortan (FBI Assistant Director)',
437 'ML': 'Martial Law',
438 'MLK': 'Martin Luther King (Civil rights advocate murdered in 1968)',
439 'MM': 'Mary McCord (Principal Deputy Assistant AG); Media Matters',
440 'MO': 'Michelle Obama (transvestite husband of Barack Obama)',
441 'MOAB': 'Mother Of All Bombs',
442 'MS': 'Microsoft; Michael Steinbach (FBI Executive Assistant Director)',
443 'MS13': 'Latino Drug Cartel; MSM',
444 'MUELLER': 'Robert Mueller (6th FBI Director)',
445 'MURKOWSKI': 'Lisa Murkowski (Sen. R-AK)',
446 'MW': 'Maxine Waters (Rep. D-CA)',
447 'MZ': 'Mark Zuckerberg (CEO of Facebook)',
448 'N&S': 'North and South',
449 'N1LB': 'No One Left Behind',
450 'No Name': 'John McCain',
451 'No Such Agency': 'NSA',
452 'NAT': 'National',
453 'NATSEC': 'National Security',
454 'NOFORN': 'No Foreign Nationals (Document Sensitivity Level)',
455 'NO NAME': 'John McCain',
456 'NO SUCH AGENCY': 'NSA',
457 'NP': 'Nancy Pelosi (Rep. D-CA); Non-Profit',
458 'NPO': 'Non-Profit Organization',
459 'NR': 'Nuclear Reactor; Nuclear Radiation',
460 'NSC': 'National Security Council',
461 'NUNES': 'Devin Nunes (Rep. R-CA)',
462 'NWO': 'New World Order; Nazi World Order (?)',
463 'NXIVM': 'Sex trafficking cult with close ties to Democratic Party',
464 'OO': 'Oval Office (White House)',
465 'OP': 'Operation; Operator; Original Post; Original Poster; Operated Plane',
466 'OPs': 'Operations',
467 'OS': 'Oversight',
468 'OWL': 'Orbital Weapon Lancet (Space-based weapon) (?)',
469 'P': 'POTUS; Presidential; Pope; Peninsula; Paragraph; Page; Payseur',
470 'P_Pers': 'POTUS Personal',
471 'PAC': 'Political Action Committee',
472 'PAGE': 'Lisa Page (FBI Special Counsel)',
473 'PANIC': 'Patriots Are Now In Control',
474 'PD': 'Police Department',
475 'PDB': 'President’s Daily Brief',
476 'PDBs': 'President’s Daily Briefs',
477 'PELOSI': 'Nancy Pelosi (Rep. D-CA)',
478 'PENCE': 'Mike Pence (VP under Trump)',
479 'PEOC': 'Presidential Emergency Operations Center',
480 'PG': 'Pizzagate/Pedogate',
481 'PL': 'Presidential Library',
482 'PM': 'Prime Minister; Post Meridiem; Paul Manafort (Campaign manager for Trump)',
483 'PODESTA': 'John Podesta (WH Chief of Staff under Clinton, Counselor under Obama)',
484 'POS': 'Piece Of Shit',
485 'POV': 'Point Of View',
486 'POVs': 'Points Of View',
487 'PP': 'Planned Parenthood',
488 'PRISM': 'NSA Internet data collection program',
489 'PS': 'Peter Strzok (FBI Lead Agent); PlayStation',
490 'PST': 'Pacific Standard Time',
491 'PTSD': 'Post-Traumatic Stress Disorder',
492 'PUTIN': 'Vladimir Putin (President of Russia)',
493 'Q&A': 'Question and Answer',
494 'Q+': 'President Trump; Q-team',
495 'R v W': 'Right versus Wrong',
496 'R\'s': 'Republicans',
497 'R’s': 'Republicans',
498 'R+D': 'Republican + Democrat',
499 'RB': 'Rachel Brand (Associate AG)',
500 'RBG': 'Ruth Bader Ginsburg (Supreme Court Associate Justice)',
501 'RC': 'Rachel Chandler (Child handler for Jeffrey Epstein who did not kill himself)',
502 'RE': 'Rahm Emanuel (White House Chief of Staff under Obama)',
503 'RED RED': 'Red Cross',
504 'RED_RED': 'Red Cross',
505 'RENEGADE': 'Secret Service codename for Barack Hussein Obama',
506 'RICE': 'Susan Rice (National Security Advisor under Obama)',
507 'RIP': 'Rest In Peace',
508 'RM': 'Robert Mueller (6th FBI Director)',
509 'RNC': 'Republican National Committee',
510 'RR': 'Rod Rosenstein (Deputy Attorney General under Trump)',
511 'RT': 'Real Time; Retweet; Rex Tillerson (Secretary of State under Trump)',
512 'RUDY': 'Rudy Giuliani (former Mayor of NYC)',
513 'SB': 'Senate Bill',
514 'SAP': 'Special Access Program',
515 'SAPs': 'Special Access Programs',
516 'Scaramucci': 'Anthony Scaramucci (WH Communications Director under Trump, fired after repeated TDS-attacks on Trump)',
517 'SCHUMER': 'Chuck Schumer (Sen. D-NY)',
518 'SCI': 'Sensitive Compartmented Information (TOP SECRET+)',
519 'SCIF': 'Sensitive Compartmented Information Facility',
520 'SDNY': 'Southern District of New York',
521 'SEC': 'Security; Section',
522 'SEC_TEST': 'Security Test',
523 'SESSIONS': 'Jeff Sessions (US Attorney General under Trump)',
524 'SH': 'Sean Hannity (Conservative TV host); Steve Huffman (CEO of Reddit)',
525 'SIGINT': 'Signals Intelligence',
526 'SIT ROOM': 'Situation Room (White House)',
527 'SM': 'Sally Moyer',
528 'SMOLLETT': 'Jussie Smollett (Hollywood actor who faked his own lynching)',
529 'SOROS': 'George Soros (Billionaire globalist investor)',
530 'SOTU': 'State Of The Union',
531 'SP': 'Samantha Power (US Ambassador to the UN)',
532 'SR': 'Seth Rich (DNC staffer murdered after leaking to Wikileaks); Susan Rice (National Security Advisor under Obama); Senior',
533 'ST': 'Shit (?)',
534 'STEELE': 'Christopher Steele (MI6 agent who concocted the Steele-dossier)',
535 'STRAT': 'Strategic',
536 'STRZOK': 'Peter Strzok (FBI agent who participated in the attempted subversion of the 2016 presidential election)',
537 'SURV': 'Surveillance',
538 'SY': 'Sally Yates (Deputy Attorney General)',
539 'TBA': 'To Be Announced',
540 'TG': 'Trey Gowdy (Rep. D-SC); Tashina Gauhar (FISA lawyer)',
541 'TM': 'Team',
542 'TP': 'Tony Podesta (Brother of John Podesta)',
543 'TRI': 'Trilateral Commission (?)',
544 'TT': 'Trump Tower; Tarmac Tapes',
545 'T-Tower': 'Trump Tower',
546 'U1': 'Uranium One',
547 'UBL': 'Usama Bin Laden',
548 'UC': 'Univerity of California',
549 'USD': 'US Dollar',
550 'USMIL': 'US Military',
551 'VIP': 'Very Important Person',
552 'VIPs': 'Very Important Persons',
553 'VJ': 'Valerie Jarret (Senior Advisor to Obama)',
554 'W&W': 'Wizards and Warlocks',
555 'WHITAKER': 'Matthew G. Whitaker (Acting US Attorney General after Sessions resigned)',
556 'WIA': 'Wounded In Action',
557 'WMDs': 'Weapons of Mass Destruction',
558 'WRAY': 'Christopher Wray (8th FBI Director)',
559 'WRWY': 'We Are With You',
560 'WW': 'World Wide; World War',
561 'WWI': 'World War 1',
562 'WWII': 'World War 2',
563 'WWIII': 'World War 3',
564 'WWG1WGA': 'Where We Go One We Go All',
565 'XKeyscore': 'NSA Internet data search and analysis tool'
566}
567
568#################################
569# Methods for Qposts Research: #
570#################################
571
572def qmap_id_to_qpost_index( qmap_id ):
573 '''<qmap_id> can either be a Qmap ID, or a tuple/list of Qmap IDs.
574 Qmap IDs are 1-based and run from oldest to latest, while Qpost index numbers are 0-based and run from latest to oldest.'''
575 if isinstance( qmap_id, ( tuple, list ) ): return [ len( _QPOSTS ) - idx for idx in qmap_id ]
576 return len( _QPOSTS ) - qmap_id
577
578
579def qpost_index_to_qmap_id( qpost_idx ):
580 '''For clarity; Same output as qmap_id_to_qpost_index( qpost_idx ).'''
581 return qmap_id_to_qpost_index( qpost_idx )
582
583
584def is_valid_qmap_id( qmap_id ):
585 '''Returns True if <qmap_id> is a valid Qmap ID number.'''
586 return qmap_id > 0 and qmap_id <= len( _QPOSTS )
587
588
589def open_qposts_json( url_qposts_json ):
590 '''Loads the specified JSON-file, and returns a list of dictionaries, or None.'''
591 # Remove user name before printing.
592 safe_url = collapse_user( url_qposts_json )
593 if os.path.exists( url_qposts_json ):
594 try:
595 with open( url_qposts_json, 'r', encoding='utf-8' ) as qps_file:
596 return json.loads( qps_file.read() )
597 except: print( f"JSON: Failed to load Qdata from file '{safe_url}'." )
598 else: print( f"Error: No such file: '{safe_url}'." )
599
600
601def download_qposts_json( url_save ):
602 '''Downloads the Qposts JSON-file from qanon.pub, and saves it to the specified url.
603 This function returns a list of dictionaries taken from the downloaded JSON-file, or None.'''
604 _URL_JSON = "https://qanon.pub/data/json/posts.json"
605 # DOWNLOAD JSON DATA FROM QANON.PUB.
606 try:
607 req = Request( _URL_JSON, headers={'User-Agent': 'Mozilla/5.0'} )
608 qdata = urlopen( req )
609 except Exception as e: print( f'{er}Failed to download json file "{_URL_JSON}":{ts} {e}' ); return None
610 try: qposts = json.loads( qdata.read().decode() )
611 except Exception as e: print( f'{er}JSON: Failed to load Qpost data from downloaded file;{ts} {e}' ); return None
612 safe_url = collapse_user( url_save ) # Hide user name before printing.
613 # DUMP DATA TO JSON FILE.
614 try:
615 with open( url_save, "w", encoding="utf-8" ) as qps_file:
616 json.dump( qposts, qps_file, ensure_ascii=False, indent=2 )
617 except Exception as e: print( f'{er}JSON: Failed to save Qpost data to file "{safe_url}":{ts} {e}' ); return None
618 return qposts
619
620
621def download_latest_qpost_nr():
622 '''Downloads a (limited) XML RSS-file from qalerts.app.
623 Returns a 2-tuple with the number of the most recent Qpost from qalerts.app, plus the root node to an XML ElementTree
624 containing (limited) records of the 20 most recent Qposts from qalerts.app; Returns (None,None) if something went wrong.
625 NB. The latest Qpost on qalerts.app is not necessarily available at exactly the same moment as the qposts.json at qanon.pub.'''
626 # import xml.etree.ElementTree as xml_tree
627 # DOWNLOAD RSS-DATA FROM QALERTS.APP
628 try:
629 _URL_RSS = "https://qalerts.app/data/rss/posts.rss"
630 req = Request( _URL_RSS, headers={ 'User-Agent': 'Mozilla/5.0' } )
631 qdata = urlopen( req )
632 except Exception: return None, None
633 root = xml_tree.fromstring( qdata.read().decode() )
634 # PARSE RSS-DATA
635 try:
636 text = root[0][10][0].text
637 return int( text.replace( 'Q Drop #', '' ) ), root
638 except: return None, root
639
640
641def check_update_local_qposts():
642 '''This function checks online if there are new Qposts available, and if so, it offers to download them.
643 The local Qposts.json file will be overwritten; if there exists a local Qposts.db database file,
644 it will be updated by adding only new records for the new Qposts.
645 Returns the updated current number of Qposts inside the local Qposts.json file.'''
646 global _QPOSTS
647 n_qposts = len( _QPOSTS )
648 # Check for latest Qposts online.
649 l_qpost, rss_root = download_latest_qpost_nr()
650 if l_qpost:
651 # NEW Qpost(s) available!
652 if l_qpost > n_qposts:
653 n_new = l_qpost - n_qposts
654 answer = input( f"{ts+tc}There {'is' if n_new == 1 else 'are'} {bw} {n_new} {ts+tc} New Qpost" +
655 f"{'' if n_new == 1 else 's'} available!{ts}\n{mc}Do you want to update your local Qposts now? (Y/n):{ts} " )
656 # Update Qposts.json, and Qposts.db if present.
657 if answer.lower() not in ['n', 'no']:
658 print( f"{tc}Updating local Qposts:{ts} Attempting to download Qposts.json from qanon.pub ..." )
659 _QPOSTS = download_qposts_json( _URL_QPOSTS )
660 n_qposts = len( _QPOSTS )
661 if os.path.exists( _URL_QPOSTS_DB ): # Update Qposts.db if present.
662 qposts_to_sqlite( _QPOSTS, _URL_QPOSTS_DB )
663 n_recs = qposts_sqlite_count_records( _URL_QPOSTS_DB )
664 elif l_qpost == n_qposts:
665 print( f'{ok}Online check completed: Your local Qposts.json is already up to date.{ts}' )
666 else: print( f'{er}Could not check online if there are any new Qposts available.{ts}' )
667 return n_qposts
668
669
670def get_qpost_media_urls( qpost ):
671 '''Returns a nested list of tuples(url,filename) for all media linked in <qpost> and in its references recursively.'''
672 tuples = []
673 if isinstance( qpost, dict ):
674 qpost_media = qpost.get( 'media', [] )
675 if qpost_media:
676 for image in qpost_media:
677 tuples.append( ( image.get( 'url', '' ), image.get( 'filename', '' ) ) )
678 # this tag is only present if the Qpost has references.
679 qpost_refs = qpost.get( 'references', [] )
680 if qpost_refs:
681 for qp_ref in qpost_refs:
682 tuples.append( get_qpost_media_urls( qp_ref ) )
683 return tuples
684
685
686def download_qpost_images( qmap_ids_list, references_too=False ):
687 '''Workaround; Tries to download the media linked in all Qposts whose Qmap ID is specified in <qmap_ids_list>;
688 This creates a subfolder for each Qmap ID that has linked media, and saves all downloaded media for that Qmap ID inside that subfolder.
689 Subfolders for downloaded media will all be created inside the folder <_URL_QPOSTS_IMAGES>, and will be named after the Qmap ID number.
690 Qposts can contain references, that can in turn also contain media. To recursively download these media too, pass <references_too>=True.
691 NB. this could result in the downloading of multiple duplicate image files, when a Qpost with images is referenced in another Qpost.
692 Statistics for 3774 Qposts:
693 excl. References: total 809 media; 694 subfolders; DL size ~370MB; DL time ~~20min; DL success-rate 100%.
694 incl. References: total 1941 media; 1273 subfolders; DL size ~864MB; DL time ~~50min; DL success-rate ~93% (Fails for 136 files).'''
695 # import os
696 def download_urls( url_list, save_folder, references_too ):
697 '''Download files from nested list <url_list> and save them into the folder <save_folder>.'''
698 urls_to_replace = [ 'https://media.8ch.net/file_store/thumb/',
699 'https://media.8ch.net/file_store/',
700 '//media.jthnx5wyvjvzsxtu.onion/file_store/' ]
701 url_replacements = [ 'https://qalerts.app/media/',
702 'https://qposts.online/assets/images/' ]
703 total_media, dl_success, already_present = 0, 0, 0
704 for item in url_list:
705 nm, ns, ap = 0, 0, 0
706 if isinstance( item, tuple ) and len(item) == 2:
707 total_media += 1
708 image_url, filename = item
709 image_url_rep = image_url
710 filename2 = os.path.split( image_url )[1] # storage name.
711 if not filename: filename = filename2
712 save_url = os.path.join( save_folder, filename )
713 if not os.path.exists( save_url ):
714
715 # Added since 8chan is offline, making all 8ch.net image-urls invalid;
716 # instead try to download the corresponding images from qposts.online or qalerts.app.
717 # TODO: change this part when 8kun has transfered all the older images from 8ch.net.
718 for url_replacement in url_replacements:
719 for url in urls_to_replace:
720 if image_url.startswith( url ):
721 image_url_rep = image_url.replace( url, url_replacement ); break
722 # Download/Save media.
723 save_ok = download_binary_file( image_url_rep, save_url )
724 # Download failed:
725 if not save_ok:
726 fparent, fname = os.path.split( image_url_rep )
727 # Retry with filename2.
728 image_url_rep = os.path.join( fparent, filename2 )
729 save_ok = download_binary_file( image_url_rep, save_url )
730 if save_ok: break
731
732 dl_success += bool( save_ok )
733 url_safe = collapse_user( save_url )
734 if save_ok: print( f"{ok}Success:{ts} media '{image_url_rep}'\n\t{ok} was saved{ts} to file '{url_safe}'." )
735 else: print( f"{er}Failed:{ts} media '{image_url_rep}'\n\t{er} was not saved{ts} to file '{url_safe}'." )
736 else: already_present += 1
737 elif isinstance( item, list ) and references_too:
738 nm, ns, ap = download_urls( item, save_folder, references_too )
739 total_media += nm; dl_success += ns; already_present += ap
740 return total_media, dl_success, already_present
741 ###### END of download_urls().
742
743 if not os.path.exists( _URL_QPOSTS_IMAGES ): os.mkdir( _URL_QPOSTS_IMAGES )
744 tuples = get_qposts_by_id( qmap_ids_list, key='', qmap_ids=True )
745 total_qposts, total_media, qp_has_media, success, already_present = 0, 0, 0, 0, 0
746 for qmap_id, qpost in tuples:
747 total_qposts += 1
748 # Nested list of tuples(url,filename).
749 qpost_media = get_qpost_media_urls( qpost )
750 qpost_own_media, qpost_ref_media = split_list( qpost_media )
751 if qpost_own_media or qpost_ref_media:
752 qp_has_media += 1
753 folder_qmap_id = os.path.join( _URL_QPOSTS_IMAGES, f'{qmap_id}', '')
754 if not os.path.exists( folder_qmap_id ):
755 # Create a subfolder for this QmapID.
756 if references_too or qpost_own_media: os.mkdir( folder_qmap_id )
757 # Download Qpost media into subfolder.
758 nm, ns, ap = download_urls( qpost_media, folder_qmap_id, references_too )
759 total_media += nm; success += ns; already_present += ap
760 n_failed = total_media - already_present - success
761 print( f"{tc}Processed {bw} {total_qposts} {tc} Qposts ({['excl.','incl.'][references_too]} references);{ts}\n{bw} {qp_has_media} {tc} " +
762 f"Qposts contain a total of {bw} {total_media} {tc} linked media:{ts}\n{tc}Media Present:{ts} {already_present}/{total_media}" +
763 f"{tc} Media Downloaded: {ok} {success}/{total_media} {tc} Download Failed: {er} {n_failed}/{total_media} {tc}.{ts}" )
764
765
766def get_qposts_by_id( qpost_ids, key='', qmap_ids=False ):
767 '''Returns a list with tuples(qmap_id,element) of all qposts whose indexes are specified in <qpost_ids>.
768 <qpost_ids>: List of indexes into the _QPOSTS list ( or list of Qmap IDs if <qmap_ids>=True ).
769 <key>: if given, the returned list elements will be only the field qpost[key]; else they will be the entire qpost dict.
770 <qmap_ids>: if True, the given <qpost_ids> are interpreted as Qmap IDs ranging from 1 to N ( where 1 is the first Qpost),
771 else they are interpreted as indexes into the _QPOSTS list, ranging from 0 to N-1 ( where 0 is the latest Qpost).'''
772 if _QPOSTS:
773 qp_count = len( _QPOSTS )
774 result = []
775 for i in qpost_ids:
776 qmap_id = i if qmap_ids else qp_count - i
777 index = qp_count - i if qmap_ids else i
778 qpost = _QPOSTS[index]
779 if key: result.append( ( qmap_id, qpost.get(key) ) )
780 else: result.append( ( qmap_id, qpost ) )
781 return result
782
783
784def search_qposts_regexp( regexp, key='' ):
785 '''Returns a list with tuples(qmap_id,qpost) of all qpost dicts whose <key>-field matches <regexp>.
786 <regexp>: String representing the regular expression to match.
787 <key>: Pass a qpost dict key, or pass "" to search the entire qpost dict as string.
788 Valid keys: see _QPOST_KEYS.'''
789 # import re
790 result = []
791 for i, qpost in enumerate( _QPOSTS ):
792 find_in = qpost[key] if qpost.get( key ) else str(qpost)
793 if re.search( regexp, find_in ): result.append( ( qpost_index_to_qmap_id(i), qpost ) )
794 return result
795
796
797def search_qposts( find, key='text' ):
798 '''Returns a list with tuples(qmap_id,qpost) of all qposts whose <key>-field contains <find>.
799 <find>: can be a complex search term using operators (and, or, not) and parentheses; complex atoms must be double-quoted.
800 If <key> is empty, it searches the text of the whole qpost dictionary for <find>; Some keys may not be present in all qposts.
801 Valid keys: see _QPOST_KEYS.'''
802 result = []
803 # class included below.
804 x = Simple_Logic_Expression( find )
805 for i, qpost in enumerate(_QPOSTS):
806 text = qpost.get( key, '' ) if key else qpost
807 if not isinstance( text, str ): text = str( text )
808 if text and x.find_in( text ):
809 result.append( ( qpost_index_to_qmap_id(i), qpost ) )
810 return result
811
812
813def search_qposts_by_date( date_condition ):
814 '''Returns a list with tuples(Qmap_id, Qpost) of all qposts whose timestamp matches <date_condition>.
815 <date_condition>: String of the format "COMP DATETIME", where COMP is one of the comparison operators in:
816 [ 'on', '>=', '<=', '!=', '==', '=', '>', '<' ], and DATETIME is a valid datetime expression such as '1 Aug 2019',
817 or a timestamp number. Each "COMP DATETIME" pair must be enclosed within double quotation marks,
818 and multiple "COMP DATETIME" pairs can be combined using logical connectives [and,or] and parentheses.
819 for example: ("> 1 Aug 2019" and "< 2 Aug 2019") or "> 1 Jan 2020" .'''
820 result = []
821 # class included below.
822 x = Simple_Logic_Expression( date_condition )
823 for i, qpost in enumerate(_QPOSTS):
824 timestamp = qpost.get( 'timestamp', -1 )
825 if not isinstance( timestamp, (int,float) ): timestamp = float( timestamp )
826 # pass function as arg.
827 if timestamp and x.match_value( timestamp, evaluate_datetime_condition ):
828 result.append( ( qpost_index_to_qmap_id(i), qpost ) )
829 return result
830
831
832def get_qposts_for_date( d ):
833 '''Returns a list of tuples(QmapID, Qpost) whose Qpost timestamp falls on the specified day <d>.
834 <d>: datetime.datetime object representing the date for which to return all Qposts that were posted on the same day.'''
835 return search_qposts_by_date( d.timestamp() )
836
837
838def get_qposts_for_dates( dates ):
839 '''Returns a list of tuples(QmapID, Qpost) whose Qpost timestamp falls on any of the dates specified in <dates>.
840 <dates>: list of datetime.datetime objects representing the dates for which to return all Qposts.'''
841 result = []
842 for d in dates:
843 result.extend( get_qposts_for_date( d ) )
844 return result
845
846
847def get_qpost_dates_for_qmap_ids( qmap_ids_list ):
848 '''Returns a list of datetime objects representing the posting dates of the Qposts specified in <qmap_ids_list>.'''
849 qposts = get_qposts_by_id( qmap_ids_list, key='timestamp', qmap_ids=True )
850 return [ datetime.datetime.fromtimestamp( float( qp[1] ), tz=None ) for qp in qposts ]
851
852
853def qclock_get_aligned_dates_for_date( qdate=datetime.datetime.today(), include_mirror=True ):
854 '''Collect earlier dates from the Qclock, that are located on the same radius or diameter line as the date specified in <qdate>.
855 <qdate>: a datetime.datetime object or a string representing the date for which to retrieve the earlier Qclock-aligned dates;
856 When passing a date string, prevent ambiguities by putting the day number before the month, and passing a 4-digit year.
857 <include_mirror>: Boolean determining whether to also include the aligned dates from the opposite side of the Qclock center.
858 Returns a tuple with 4 elements: (the parsed input date; a standard string representation of the parsed input date; a list containing
859 the aligned dates on this side of the center; and a list containing the aligned dates on the opposite side of the center: this latter
860 list will be empty if <include_mirror>=False).'''
861 # import datetime; _DT_FORMAT_L = '%A %d %B %Y'
862 if isinstance( qdate, str ): dt, dts, _ = parse_datetime_string( qdate, _DT_FORMAT_L )
863 elif isinstance( qdate, datetime.datetime ): dt, dts = qdate, qdate.strftime( _DT_FORMAT_L )
864 if not dt: return None, '', [], []
865 # Serial Day Number of input date.
866 else: d_target = dt.toordinal()
867 aligned, aligned_mirror, current, mirror = [], [], d_target, d_target - 30
868 # QClock Start: '10-28-17'; ( hour, angle, minutes ) = ( 4, 10, 20 ) = DJT.
869 d_start = 736630
870 while current >= d_start:
871 aligned.append( datetime.datetime.fromordinal( current ) ); current -= 60
872 if include_mirror:
873 while mirror >= d_start:
874 aligned_mirror.append( datetime.datetime.fromordinal( mirror ) ); mirror -= 60
875 return dt, dts, aligned, aligned_mirror
876
877
878def qclock_get_aligned_dates_for_clocktime( qtime=datetime.datetime.now(), round_to_nearest=True ):
879 '''Collect all dates between 10-28-2017 and today, that are located on either of the Qclock hands at the specified clock time.
880 <qtime>: A datetime.datetime, datetime.time, or a 2-tuple(H,M) specifying the digital time for which to retrieve the aligned dates.
881 NB. For a digital time of 04:20 the hour-hand on the analog clock is pointing at precisely 21.67 minutes, which would be
882 returned here as the 22nd minute if <round_to_nearest>=True, else as the 21st minute;
883 Returns a tuple with 2 lists containing datetime.datetime objects: the first list contains the dates located on the hour-hand, and the
884 second list contains the dates located on the minute-hand of the Qclock.'''
885 # import datetime
886 hour_hand, minute_hand, roundoff = [], [], [int,round][bool(round_to_nearest)]
887 d_today = datetime.datetime.today().toordinal()
888 # QClock start date: '10-28-2017'
889 d_start = 736630
890 if isinstance( qtime, ( datetime.datetime, datetime.time ) ): H, M = qtime.hour, qtime.minute
891 elif isinstance( qtime, (tuple,list) ) and len( qtime ) > 1: H, M = qtime[0], qtime[1]
892 else: return [], []
893 try: H, M = int( H ) % 12, int( M ) % 60
894 except: return [], []
895 H_date = d_start + roundoff( ( ( H + 8 ) % 12 + M / 60 ) * 5 ) # rounded to floor or nearest.
896 M_date = d_start + ( M + 40 ) % 60
897 while H_date <= d_today:
898 hour_hand.append( datetime.datetime.fromordinal( H_date ) ); H_date += 60
899 while M_date <= d_today:
900 minute_hand.append( datetime.datetime.fromordinal( M_date ) ); M_date += 60
901 return hour_hand, minute_hand
902
903
904def qclock_get_aligned_qposts_for_date( qdate=datetime.datetime.today(), include_mirror=True, print_list=True ):
905 '''Collects all earlier Qposts posted on a date that aligns with the given <qdate> on the Qclock.
906 <qdate>: a datetime.datetime object or a string representing the date for which to retrieve all earlier aligned Qposts.
907 <include_mirror>: Boolean determining whether to also include the aligned Qposts from the opposite side of the Qclock center.
908 <print_list>: If True, it prints out the aligned dates and the number of Qposts that were posted on each of those dates.
909 Returns a single list of tuples(QmapID, Qpost) for all Qposts whose date aligns on the Qclock with the input date.'''
910 # import datetime
911 dt, dts, aligned, mirror = qclock_get_aligned_dates_for_date( qdate, include_mirror )
912 if dts:
913 result, s_aligned, s_mirror = [], [], []
914 for d in aligned:
915 qposts_for_day = get_qposts_for_date( d ); n_qp = len(qposts_for_day)
916 result.extend( qposts_for_day )
917 if print_list: s_aligned.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
918 if print_list: print( f'{tc}Qclock Directly aligned dates for {bw} {dts} {ts+tc}:{ts}\n{", ".join( s_aligned )}' )
919 if include_mirror:
920 for d in mirror:
921 qposts_for_day = get_qposts_for_date( d ); n_qp = len(qposts_for_day)
922 result.extend( qposts_for_day )
923 if print_list: s_mirror.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
924 if print_list: print( f'{tc}Qclock Opposite aligned dates for {bw} {dts} {ts+tc}:{ts}\n{", ".join( s_mirror )}' )
925 return result
926
927
928def qclock_get_aligned_qposts_for_clocktime( qtime, round_to_nearest=True, print_list=True ):
929 '''Returns a list of tuples(QmapID, Qpost) for all Qposts whose date is located on one of the hands of the Qclock on the given clocktime.
930 <qtime>: A datetime.datetime, datetime.time, or a 2-tuple(H,M) specifying the digital time for which to retrieve the Qclock-aligned Qposts.
931 <print_list>: If True, it prints out the dates on the Qclock-hands, and the number of Qposts that were posted on each of those dates.'''
932 result, s_hour, s_minute = [], [], []
933 hour_hand, minute_hand = qclock_get_aligned_dates_for_clocktime( qtime, round_to_nearest )
934 for d in hour_hand:
935 qposts_for_day = get_qposts_for_date( d ); n_qp = len( qposts_for_day )
936 result.extend( qposts_for_day )
937 if print_list: s_hour.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
938 for d in minute_hand:
939 qposts_for_day = get_qposts_for_date( d ); n_qp = len( qposts_for_day )
940 if hour_hand != minute_hand: result.extend( qposts_for_day )
941 if print_list: s_minute.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
942 if print_list:
943 print( f'{tc}Qclock Hour-hand dates for clocktime ({qtime[0]:02d}:{qtime[1]:02d}):{ts}\n{", ".join( s_hour )}' )
944 print( f'{tc}Qclock Minute-hand dates for clocktime ({qtime[0]:02d}:{qtime[1]:02d}):{ts}\n{", ".join( s_minute )}' )
945 return result
946
947
948def qclock_get_aligned_qposts_for_qmap_id( qmap_id, include_mirror=True, print_list=True ):
949 '''Collects all earlier Qposts aligning on the Qclock with the Qpost of the given <qmap_id>.
950 <qmap_id>: Integer Qmap ID number of the Qpost for which to get all Qclock-aligned Qposts.
951 <include_mirror>: Boolean determining whether to also include the aligned dates from the opposite side of the Qclock center.
952 <print_list>: If True, it prints out the aligned dates and the number of Qposts that were posted on each of those dates.
953 Returns a list of tuples(QmapID, Qpost) for all Qposts whose date aligns on the Qclock with the post date of the given <qmap_id>.'''
954 return qclock_get_aligned_qposts_for_date( get_qpost_dates_for_qmap_ids( [ qmap_id ] )[0], include_mirror, print_list )
955
956
957def qposts_to_sqlite( qposts, url_save ):
958 '''Exports a list of Qpost-dictionaries into an SQLite3 database; Only non-existing (new) records are added to the database.'''
959 # import os, sqlite3
960 try:
961 qposts_reversed = qposts.copy()
962 # Reverse order of Qposts, so that the oldest Qpost gets qmap_id=1.
963 qposts_reversed.reverse()
964 # Creates an empty db if the specified name is not found.
965 connection = sqlite3.connect( url_save )
966 cursor = connection.cursor()
967 success = 0
968 str_sql = "CREATE TABLE IF NOT EXISTS qposts (qmap_id INTEGER PRIMARY KEY, timestamp INTEGER, text TEXT, media TEXT, "
969 str_sql += "refs TEXT, name TEXT, trip TEXT, userId TEXT, link TEXT, source TEXT, threadId TEXT, id TEXT, title TEXT, "
970 str_sql += "subject TEXT, email TEXT, timestampDeletion INTEGER, CONSTRAINT unique_id_ts UNIQUE (id,timestamp));"
971 # Create table "qposts" if not already present.
972 cursor.execute( str_sql )
973 connection.commit()
974 # Add only the *new* qposts to the table.
975 for qp in qposts_reversed:
976 values = ( None, int( qp.get( 'timestamp', -1 ) ), qp.get( 'text', '' ), str( qp.get( 'media', '' ) ),
977 str( qp.get( 'references', '' ) ), qp.get( 'name', '' ), qp.get( 'trip', '' ), qp.get( 'userId', '' ),
978 qp.get( 'link', '' ), qp.get( 'source', '' ), qp.get( 'threadId', '' ), qp.get( 'id', '' ),
979 qp.get( 'title', '' ), qp.get( 'subject', '' ), qp.get( 'email', '' ), qp.get( 'timestampDeletion', None ) )
980 try: cursor.execute( "INSERT INTO qposts VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );", values ); success += 1
981 except: pass
982 connection.commit()
983 connection.close()
984 filename = os.path.split( url_save )[1]
985 # hlink = ansi_hyperlink( f'file://{url_save}', filename, fg=(150,190,10), bg=(), style=(0,1,1,0,0) )
986 hlink = f'file://{url_save}'
987 print( f'{tc}Exported {bw} {success} {ts+tc} new Qpost records to SQLite3 Database "{hlink}{tc}".{ts}' )
988 except Exception as e:
989 # Hide user name before printing url.
990 safe_url = collapse_user( url_save )
991 print( f'{er}Error while exporting Qpost records to SQLite3 Database "{safe_url}":{ts} {e}' )
992
993
994def qposts_sqlite_query( url_db, sql='SELECT * FROM qposts;' ):
995 '''Executes a SELECT query <sql> in the SQLite3 database <url_db>, and returns the result.'''
996 # import os, sqlite3
997 result = '<Error>'
998 if os.path.exists( url_db ):
999 try:
1000 connection = sqlite3.connect( url_db )
1001 cursor = connection.cursor()
1002 result = list( cursor.execute( sql ) )
1003 connection.close()
1004 except:
1005 # Hide user name before printing url.
1006 safe_url = collapse_user( url_db )
1007 print( f'{er}SQL: Failed to execute query "{sql}" in SQLite3 Database "{safe_url}".{ts}' )
1008 return result
1009
1010
1011def qposts_sqlite_count_records( url_db, table='qposts' ):
1012 '''Returns the number of records in the table <table> of the SQLite3 database <url_db>.'''
1013 rec_count = 0
1014 try:
1015 connection = sqlite3.connect( url_db )
1016 cursor = connection.cursor()
1017 cursor_ids = cursor.execute( f"SELECT * FROM {table};" )
1018 rec_count = len( list( cursor_ids ) )
1019 connection.close()
1020 except: print( f'{er}Failed to count records in SQLite3 Database table "{table}".{ts}' )
1021 return rec_count
1022
1023
1024def qpost_text_cleanup( qp_text ):
1025 '''Cleanup leftover Html-formatting from the Qpost text field <qp_text>.'''
1026 # import re, html
1027 if qp_text:
1028 # remove single newline character at the end.
1029 if qp_text[-1] == '\n': qp_text = qp_text[:-1]
1030 # strong-tags --> bold.
1031 qp_text = re.sub( r'<strong>(.*?)</strong>', r'[1m\1[0m', qp_text )
1032 # em-tags --> italic.
1033 qp_text = re.sub( r'<em>(.*?)</em>', r'[3m\1[23m', qp_text )
1034 # u-tags --> underline.
1035 qp_text = re.sub( r'<u>(.*?)</u>', r'[4m\1[24m', qp_text )
1036 # headings --> red bold.
1037 qp_text = re.sub( r'<span class="heading">(.*?)</span>', r'[1;38;2;175;10;15m\1[0m', qp_text )
1038 # spoilers --> dimmed text.
1039 qp_text = re.sub( r'<span class="spoiler">(.*?)</span>', r'[2m\1[22m', qp_text )
1040 # detected --> inverted color.
1041 qp_text = re.sub( r'<span class="detected">(.*?)</span>', r'[7m\1[27m', qp_text )
1042 #body-line empty --> plain text.
1043 qp_text = re.sub( r'<p class="body-line empty ">(.*?)</p>', r'\1', qp_text )
1044
1045 # remove faulty tag.
1046 qp_text = re.sub( r'</p><p class="body-line ltr quote">', '', qp_text )
1047 qp_text = re.sub( r'</p><p class="body-line ltr ">', '', qp_text )
1048 # Remove HTML escape-codes ( requires Python 3.4 )
1049 return html.unescape( qp_text )
1050
1051
1052def print_qpost( qp, lv='' ):
1053 '''Print elements of the specified Qpost.
1054 # <qp> : dictionary representing a Qpost from Qanon.pub.
1055 # <lv> : string representing the submenu depth level: pass _LVL_INDENT per added sublevel.'''
1056
1057 # TIMESTAMP
1058 qp_timestamp = qp.get( 'timestamp', '' )
1059 qp_date = datetime.datetime.fromtimestamp( qp_timestamp )
1060 qp_datestr = qp_date.strftime( _DTM_FORMAT )
1061 qp_datediff = datetime.datetime.now() - qp_date
1062 print( f'{lv}{tc}Timestamp: {ts}{qp_timestamp}\t{tc}Date: {ts}{qp_datestr}' )
1063 print( f'{lv}{tc}Ago: {ts}{qp_datediff}' )
1064 qp_ts_delete = qp.get( 'timestampDeletion', None )
1065 if qp_ts_delete:
1066 qp_ts_datestr = datetime.datetime.fromtimestamp( qp_ts_delete ).strftime( _DTM_FORMAT )
1067 print( f'{lv}{tc}Timestamp Deletion: {ts}?{qp_ts_delete}\t{tc}Date: {ts}{qp_ts_datestr}' )
1068
1069 # NAME
1070 qp_name = qp.get( 'name', '<None>' )
1071 qp_trip = qp.get( 'trip', '<None>' )
1072 qp_userId = qp.get( 'userId', '<None>' )
1073 print( f'{lv}{tc}Name: {ts}{qp_name}\t\t\t{tc}Tripcode: {ts}{qp_trip}\t{tc}User ID: {ts}{qp_userId}' )
1074
1075 # SOURCE
1076 qp_source = qp.get( 'source', '' )
1077 # this tag can be missing in some posts.
1078 qp_threadId = qp.get( 'threadId', '' )
1079 qp_id = qp.get( 'id', '' )
1080 qp_link = qp.get( 'link', '' )
1081 print( f'{lv}{tc}Source: {ts}{qp_source}\t{tc}Thread ID: {ts}{qp_threadId}\t{tc}Post ID: {ts}{qp_id}' )
1082 print( f'{lv}{tc}Source Link: {ts}{qp_link}' )
1083
1084 #TITLE/SUBJECT
1085 for title in [ 'subject', 'title' ]:
1086 qp_title = qp.get( title, None )
1087 if qp_title is not None: print( f'{lv}{tc}{title.title()}: {ts}{qp_title}' )
1088
1089 # TEXT
1090 qp_text = qp.get( 'text', '' )
1091 print( f'{lv}{tc}Text: {ts}' )
1092 if qp_text:
1093 qp_text = qpost_text_cleanup( qp_text )
1094
1095 if _MAX_WRAP > 0:
1096 lines = []
1097 # wrap text to a fixed width.
1098 for txline in qp_text.splitlines():
1099 lines.extend( wrap( txline, _MAX_WRAP, break_long_words=False, break_on_hyphens=False, initial_indent=lv, subsequent_indent=lv ) )
1100 qp_text = '\n'.join( lines )
1101
1102 if _SHOW_ABBR_TOOLTIPS:
1103 # add Tooltip to known abbreviations.
1104 for abbr in _QPOST_ABBR:
1105 if abbr in qp_text:
1106 qp_text = re.sub( r"(\W|\n|^)" + re.escape(abbr) + r"(\W|\n|$)", r"\1" +
1107 ansi_hyperlink( _QPOST_ABBR[abbr], abbr ) + r"\2", qp_text )
1108
1109 print( f'{qp_text}' )
1110
1111 # MEDIA
1112 qp_media = qp.get( 'media', [] )
1113 if qp_media is None: qp_media = []
1114 print( f'{lv}{tc}Media: {ts}{len(qp_media)}' )
1115 for q_pic in qp_media:
1116 print( f"{lv}{_LVL_INDENT} {q_pic['url']}" )
1117 print( f"{lv}{_LVL_INDENT} {q_pic['filename']}" )
1118
1119 # REFERENCES
1120 # this tag is only present if the Qpost has references.
1121 qp_refs = qp.get( 'references', [] )
1122 print( f'{lv}{tc}References: {ts}{len(qp_refs)}' )
1123 for q_ref in qp_refs:
1124 print_qpost( q_ref, lv + _LVL_INDENT )
1125
1126
1127def find_regexp_groups( tuples, regexp ):
1128 '''Returns a list of matched groups from <regexp>.
1129 <tuples>: List of tuples(QmapID,str_or_dict).
1130 <regexp>: The regular expression to match in the 2nd element of <tuples>; parenthesized groups are returned.
1131 This argument should be passed as a raw string, e.g. fr"{regexp}".'''
1132 #import re
1133 ret = []
1134 for item in tuples:
1135 s = re.findall( regexp, str(item[1]) )
1136 if s and len( s ) > 0: ret.append( (item[0], s) )
1137 return ret
1138
1139
1140def print_qpost_tuples( qp_tuples, key='' ):
1141 '''Display results returned by search_qposts(), search_qposts_by_date(), get_qposts_by_id().'''
1142 for qp in qp_tuples:
1143 print( f'\n{tc}Qmap ID:{ts} {qp[0]}' )
1144 if isinstance( qp[1], dict ): print_qpost( qp[1] )
1145 else: print( f"{tc}{key}:{ts} {qp[1] if key != 'text' else qpost_text_cleanup(qp[1]) }" )
1146
1147
1148def print_unique_qpost_field_values( qp_tuples, key='trip' ):
1149 '''Display unique values found in the Qpost field <key> of the Qposts in <qp_tuples>.'''
1150 qp_unique = set( [ qp[1] for qp in qp_tuples ] )
1151 print( f'{tc}Unique values for key=\'{key}\':{ts} {qp_unique}' )
1152
1153
1154def print_qpost_field_frequency_list( qp_tuples, key='', lex=0 ):
1155 '''Display a list of character/word frequencies found in the Qpost field <key> of the Qposts in <qp_tuples>.'''
1156 qp_freqlist = [ ( qp[0], count_frequencies( qp[1], lex=lex ) ) for qp in qp_tuples ]
1157 print( f"{tc}{('Character','Word')[min(max(0,lex),1)]} Frequency list for key=\'{key}\':{ts} {qp_freqlist}" )
1158
1159
1160def qposts_terminal_loop():
1161 '''Start an input loop in the Terminal where the user can perform various operations on the _QPOSTS list.'''
1162 global _QMAP_IDS, _QPOSTS
1163 n_posts = len(_QPOSTS)
1164 choice = 0
1165 duration_options = [ (f' {tc}1{ts}: as a number of seconds.', ['1'], ''),
1166 (f' {tc}2{ts}: in the format (H)HH:MM:SS.', ['2'], ''),
1167 (f' {tc}3{ts}: in the format DAYS, HH:MM:SS.', ['3'], ''),
1168 (f' {tc}4{ts}: in the format 1w2d3h4m5s.', ['4'], ''),
1169 (f' {tc}5{ts}: as a verbose duration string.', ['5'], '') ]
1170 sb_overwrite = f'{cu+gr}Search results will overwrite the current subset.{ts}'
1171 options = [ (f' {tc+bd}A) {ts+tc}CURRENT SUBSET{ts}', 'A'),
1172 (f' {tc}1{ts}: Define a new subset of Qmap IDs.', ['1'], 'A'),
1173 (f' {tc}2{ts}: Display the current subset of {bw}' + ' {} ' + f'{ts} Qmap IDs.', ['2'], 'A'),
1174 (f' {tc}3{ts}: Display (a field of) all Qposts from the current subset.', ['3'], 'A'),
1175 (f' {tc}4{ts}: Display unique values from a field of all Qposts from the current subset.', ['4'], 'A'),
1176 (f' {tc}5{ts}: Display a list of formatted datetimes of all Qposts from the current subset.', ['5'], 'A'),
1177 (f' {tc}6{ts}: Display a list of relative time-intervals between the Qposts from the current subset.', ['6'], 'A'),
1178 (f' {tc}7{ts}: Display a list of character/word-frequencies for Qposts from the current subset.', ['7'], 'A'),
1179 (f' {tc}8{ts}: Find matching Regular Expression groups in the Qposts from the current subset.', ['8'], 'A'),
1180 (f' {tc}9{ts}: Find the longest common substring in (a field of) all Qposts from the current subset.', ['9'], 'A'),
1181 (f' {tc+bd}B) {ts+tc}SEARCH ALL QPOSTS{ts}', 'B'),
1182 (f'{tc}10{ts}: Case-sensitive text search in (a field of) all {bw}' + ' {} ' + f'{ts} Qposts; {sb_overwrite}', ['10'], 'B'),
1183 (f'{tc}11{ts}: Find all Qposts matching a Regular Expression; {sb_overwrite}', ['11'], 'B'),
1184 (f'{tc}12{ts}: Date/Time search in all Qposts; {sb_overwrite}', ['12'], 'B'),
1185 (f'{tc}13{ts}: Find all (earlier) Q-Clock aligned Qposts for a given date; {sb_overwrite}', ['13'], 'B'),
1186 (f'{tc}14{ts}: Find all (earlier) Q-Clock aligned Qposts for a given Qmap ID; {sb_overwrite}', ['14'], 'B'),
1187 (f'{tc}15{ts}: Find all Q-Clock aligned Qposts for a given clock time (HH:MM); {sb_overwrite}', ['15'], 'B'),
1188 (f' {tc+bd}C) {ts+tc}DOWNLOAD / SAVE{ts}', 'C'),
1189 (f'{tc}16{ts}: Check online if there are new Qposts available.', ['16'], 'C'),
1190 (f'{tc}17{ts}: Download the latest Qposts JSON-file from qanon.pub to your Downloads folder.', ['17'], 'C'),
1191 (f'{tc}18{ts}: Export all (new) Qposts to an SQLite3 Database inside your Downloads folder.', ['18'], 'C'),
1192 (f'{tc}19{ts}: Download images from the media/image URLs from all Qposts in the current subset.', ['19'], 'C') ]
1193 db_options = [ (f' {tc+bd}D) {ts+tc}SQLITE3 QPOSTS DATABASE{ts}', 'D'),
1194 (f'{tc}20{ts}: Display the results of an SQL SELECT-query from your Qposts Database.', ['20'], 'D') ]
1195 message = [ f'\n{tc+iv+cu}Qposts research tools menu:{ts}',
1196 f'{mc}Please choose one of the menu options ( or q to quit ):{ts} ',
1197 f'{tc}Type one of the commands: ' +
1198 f_all( ['all','first N','last N','prev N','next N','reverse','sort up','sort down'], f'{mc+iv}', f'{ts}', ' ' ) +
1199 f"{tc},{ts}\n{tc}or type a series of Qmap IDs and/or Qmap ID subranges separated by comma's,{ts}\n" +
1200 f"{tc}where Qmap ID subranges can be specified by placing a dash '{mc+iv}-{ts+tc}' in between two Qmap IDs,{ts}\n" +
1201 f"{tc}and where an asterisk '{mc+iv}*{ts+tc}' represents the most recent Qmap ID, for example:{ts+gr} 5-8,2777-*{ts}\n" +
1202 f"{mc}Please enter a new subset of Qmap IDs:{ts} ",
1203 f"{er}Incorrect list format: should only be integers separated by comma's, for example:{ts} 1,2,82,14",
1204 f'{er}Incorrect Qmap ID: the numbers should be from 1 to {n_posts}.{ts}',
1205 f'{er}Incorrect range format: should be 2 integers separated by a hyphen, for example:{ts} 1-10',
1206 f'{tc+cu}Fields:{ts} ' + f_all( _QPOST_KEYS, f'{mc+iv}', f'{ts}', ' ' ) +
1207 f'{ts}\n{mc}Please enter a field name/number (or nothing for the whole record):{ts} ',
1208 f'{tc+cu}Current subset of Qmap IDs:{ts}',
1209 f'{tc}Search terms can be combined using the keywords ' + f_all( ['and','or','not'], f'{mc+iv}', f'{tc}') +
1210 f' and using {mc+iv}(parentheses){tc}.{ts}\n{tc}Atoms containing spaces, keywords or parentheses ' +
1211 f'should be enclosed in {mc+iv}"double quotation marks"{tc}.{ts}\n' +
1212 f'{mc}Please enter a Case-sensitive search term:{ts} ',
1213 f'{tc}Date search terms must have the format: "OP DATETIME" (including quotation marks),{ts}\n' +
1214 f'{tc}where OP is one of the comparison operators ' + f_all( ['on','>=','<=','!=','=','>','<'], f'{mc+iv}', f'{tc}' ) +
1215 f',{ts}\n{tc}and where DATETIME is either a timestamp or a verbose date string like "28 Oct 2017".{ts}\n' +
1216 f'{tc}Both parts "OP DATETIME" together must be enclosed in {mc+iv}"double quotation marks"{tc}.{ts}\n' +
1217 f'{tc}Date Search terms can be combined using the keywords ' + f_all( ['and','or','not'], f'{mc+iv}', f'{tc}') +
1218 f', and using {mc+iv}(parentheses){tc}.{ts}\n{tc}The {mc+iv}on{tc} operator selects all posts from the same day, ' +
1219 f'for example:{ts} \"on 11 nov 2019\"\n{mc}Please enter a date search term:{ts} ',
1220 f'{tc+cu}Relative durations between Qposts:{ts}',
1221 f'{tc+cu}Formatted posting datetimes:{ts}',
1222 f'{tc}Interval Durations can be represented in one of the following formats:{ts}',
1223 f'{mc}Please enter a date after 28 October 2017:{ts} ',
1224 f'{mc}Please enter a valid Qmap ID number:{ts} ',
1225 f'{cu+gr} ? See ' + ansi_hyperlink(_DOC_STRFTIME,"strftime format codes",style=(0,1,1,0,0)) + f':\n' +
1226 f' {mc+iv}%d{tc} = day number (01 to 31); {mc+iv}%U{tc} = week number (00 to 53); {ts}\n' +
1227 f' {mc+iv}%A{tc} = weekday; {mc+iv}%a{tc} = weekday abbr.; {mc+iv}%w{tc} = weekday number (0 to 6); {ts}\n' +
1228 f' {mc+iv}%B{tc} = month; {mc+iv}%b{tc} = month abbr.; {mc+iv}%m{tc} = month number (01 to 12); {ts}\n' +
1229 f' {mc+iv}%y{tc} = year number (00 to 99); {mc+iv}%Y{tc} = year number (0000 to 9999);{ts}\n' +
1230 f' {mc+iv}%H{tc} = hour (00 to 23); {mc+iv}%I{tc} = hour (01 to 12); {mc+iv}%p{tc} = "AM" or "PM";{ts}\n' +
1231 f' {mc+iv}%M{tc} = minute (00 to 59); {mc+iv}%S{tc} = second (00 to 59); {mc+iv}%s{tc} = timestamp; {ts}\n' +
1232 f' {mc+iv}%c{tc} = datetime; {mc+iv}%x{tc} = date; {mc+iv}%X{tc} = time. {ts}\n' +
1233 f'{tc} {cu}The default format string is a long datetime: {mc+iv}{_DTM_FORMAT}{ts}\n' +
1234 f'{mc}Please enter a valid strftime format string (or nothing for default):{ts} ',
1235 f'{mc}Please enter a digital clocktime Hour and Minute (H:M):{ts} ',
1236 f'{tc}NB. at' + ' {} digital time, the Hour-hand on the analogue Qclock points to the {}-minute mark{},' +
1237 f'{ts}\n{tc}and forms a' + ' {}-degree angle with the Minute-hand.' + f'{ts}',
1238 f'{tc}The symbol {mc+iv}^{tc} matches the start of the text, {mc+iv}${tc} matches the end of the text, and {mc+iv}\\n{tc} ' +
1239 f'matches a newline character.{ts}\n{tc}Matched groups can be captured within parentheses, e.g.{ts} These people are (.*)\\n\n' +
1240 f'{mc}Please enter a Regular Expression ( capturing groups ):{ts} ',
1241 f'{mc}Please enter a Regular Expression pattern to find in all Qposts:{ts} ',
1242 f'{mc}Enter the lexical element to count: 0 = characters, or 1 = words:{ts} ',
1243 f'{mc}Also download images from referenced posts? [Y/n]:{ts} ',
1244 f'{tc} Database :{ts} ' + ansi_hyperlink( f'file://{_URL_QPOSTS_DB}', _SAFE_URL_QPOSTS_DB, fg=(150,190,10) ) + '\n' +
1245 f'{tc} Table name :{ts} {mc+iv}qposts{ts}\n' +
1246 f'{tc} Field names :{ts} ' + f_all( _QPOST_DB_COLS, f'{mc+iv}', f'{ts}', ' ') + f'\n' +
1247 f'{tc} Query Format:{ts} {mf}SELECT {cu}<fields>{mf} FROM {cu}qposts{mf} WHERE {cu}<condition>{mf};{ts}\n' +
1248 f"{tc} For Example :{ts} SELECT qmap_id, text FROM qposts WHERE strftime( '%w/%m', timestamp, 'unixepoch')='4/05';\n" +
1249 f"{' '*15}{cu+gr}( This example above finds all Qposts that are posted on a Friday in May ){ts}\n" +
1250 f"{' '*15}{cu+gr} See " + ansi_hyperlink(_DOC_SQLITE_SLCT,'SQLite Select',style=(0,1,1,0,0)) +
1251 f'{cu+gr} (online) for more information about SQLite SELECT queries.{ts}\n' +
1252 f"{' '*15}{cu+gr} See " + ansi_hyperlink(_DOC_SQLITE_FUNC,'SQL Functions',style=(0,1,1,0,0)) +
1253 f'{cu+gr} (online) for more functions that can be used inside the query.{ts}\n' +
1254 f'{mc}Please enter a valid SQL SELECT query:{ts} ',
1255 f'{er}Incorrect SELECT Query: must start with the word SELECT followed by a space and a value.{ts}' ]
1256 def input_qpost_key( default='' ):
1257 '''Ask user to enter a valid qpost field key; else it returns the given default.'''
1258 k, answer = 0, input( message[6] )
1259 if answer.isdigit(): k = int( answer )
1260 return _QPOST_KEYS[k-1] if k > 0 and k <= len( _QPOST_KEYS ) else ( answer if answer in _QPOST_KEYS else default )
1261 def handle_search_results( tuples, key, print_tuples=True ):
1262 '''Display the search results specified by <tuples> and <key>; returns the new subset of Qmap IDs.'''
1263 # construct Current Subset.
1264 qmap_ids = [ qpt[0] for qpt in tuples ] if tuples else []
1265 if print_tuples: print_qpost_tuples( tuples, key )
1266 print( message[7], integer_list_to_range_string( qmap_ids ) )
1267 return qmap_ids
1268 while choice not in ['_EXIT','_ERROR']:
1269 menu = options.copy()
1270 # insert current subset size.
1271 menu[2] = (options[2][0].format( len(_QMAP_IDS) ), options[2][1], options[2][2])
1272 # insert current number of Qposts.
1273 menu[11] = (options[11][0].format( len(_QPOSTS) ), options[11][1], options[11][2])
1274 if os.path.exists( _URL_QPOSTS_DB ): menu.extend( db_options )
1275 choice = input_menu( menu, message[0], message[1], visible=_VISIBLE_MENU_GROUPS )
1276 #(1) DEFINE A SUBSET OF QMAP IDS:
1277 if choice in menu[1][1]:
1278 range_string = input( message[2] )
1279 rs_lower = range_string.lower()
1280 if rs_lower in [ 'reverse', 'rev' ]: _QMAP_IDS.reverse()
1281 elif rs_lower in [ 'sort down', 'sort desc', 'desc' ]: _QMAP_IDS.sort( reverse=True )
1282 elif rs_lower in [ 'sort up', 'sort asc', 'sort', 'asc' ]: _QMAP_IDS.sort()
1283 else:
1284 range_string = description_to_range_string( range_string, len(_QPOSTS), _QMAP_IDS )
1285 _QMAP_IDS = range_string_to_integer_list( range_string, validate=is_valid_qmap_id, msgs=message[3:6] )
1286 print( message[7], integer_list_to_range_string( _QMAP_IDS ) )
1287 #(2) DISPLAY CURRENT SUBSET OF QMAP IDS:
1288 elif choice in menu[2][1]:
1289 print( message[7], integer_list_to_range_string( _QMAP_IDS ) )
1290 #(3) DISPLAY QPOSTS (or FIELDS) FROM THE CURRENT SUBSET:
1291 elif choice in menu[3][1]:
1292 answer = input_qpost_key()
1293 tuples = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
1294 print_qpost_tuples( tuples, answer )
1295 #(4) DISPLAY UNIQUE VALUES FROM A FIELD IN THE QPOSTS FROM THE CURRENT SUBSET:
1296 elif choice in menu[4][1]:
1297 answer = input_qpost_key( 'trip' )
1298 tuples = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
1299 print_unique_qpost_field_values( tuples, answer )
1300 #(5) DISPLAY FORMATTED POSTING DATETIMES OF QPOSTS FROM THE CURRENT SUBSET:
1301 elif choice in menu[5][1]:
1302 strftime_format = input( message[15] )
1303 if strftime_format == '': strftime_format = _DTM_FORMAT
1304 dates = get_qpost_dates_for_qmap_ids( _QMAP_IDS )
1305 print( message[11], [ d.strftime( strftime_format ) for d in dates ] )
1306 #(6) DISPLAY RELATIVE TIME INTERVALS BETWEEN QPOSTS FROM THE CURRENT SUBSET:
1307 elif choice in menu[6][1]:
1308 tuples = get_qposts_by_id( _QMAP_IDS, key='timestamp', qmap_ids=True )
1309 i_option = input_menu( duration_options, message[12], message[1] )
1310 previous, durations = 0, []
1311 for i,tstamp in enumerate( tuples ):
1312 if i == 0: previous = tstamp[1]; durations.append( ( tstamp[0], 0 ) )
1313 else:
1314 difference = tstamp[1] - previous
1315 if i_option == '1' : durations.append( ( tstamp[0], difference ) )
1316 elif i_option in ['2','3'] : durations.append( ( tstamp[0], seconds_to_HMS( difference, days=(i_option == 3) ) ) )
1317 elif i_option == '4' : durations.append( ( tstamp[0], seconds_to_timestring( difference, separator='' ) ) )
1318 elif i_option == '5' : durations.append( ( tstamp[0], seconds_to_timestring( difference, [], True, ', ' ) ) )
1319 previous = tstamp[1]
1320 print( message[10], durations )
1321 #(7) DISPLAY CHAR/WORD FREQUENCIES IN THE CURRENT SUBSET:
1322 elif choice in menu[7][1]:
1323 lex = input( message[20] )
1324 lex = 0 if lex not in ['0','1'] else int( lex )
1325 answer = input_qpost_key( 'text' )
1326 tuples = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
1327 print_qpost_field_frequency_list( tuples, key=answer, lex=lex )
1328 #(8) DISPLAY MATCHING REGEXP GROUPS IN THE CURRENT SUBSET:
1329 elif choice in menu[8][1]:
1330 regexp = input( message[18] )
1331 answer = input_qpost_key()
1332 tuples = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
1333 ret = find_regexp_groups( tuples, fr'{regexp}' )
1334 print( f'{tc}Matched Groups for key=\'{answer}\':{ts} ', ret )
1335 #(9) FIND LONGEST COMMON SUBSTRING IN THE CURRENT SUBSET:
1336 elif choice in menu[9][1]:
1337 answer = input_qpost_key()
1338 tuples = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
1339 str_lcs = longest_common_substring( [ str(qpt[1]) for qpt in tuples ] )
1340 print( f'{tc}Longest Common Substring for key=\'{answer}\':{ts} "{str_lcs}"' )
1341 #(10) SEARCH SUBSTRING IN ALL QPOSTS, CREATING A NEW SUBSET:
1342 elif choice in menu[11][1]:
1343 search_term = input( message[8] )
1344 answer = input_qpost_key()
1345 tuples = search_qposts( search_term, key=answer )
1346 print( f'{tc}Found {bw} {len(tuples)} {tc} Qposts matching search term="{search_term}".{ts}' )
1347 _QMAP_IDS = handle_search_results( tuples, answer )
1348 #(11) FIND QPOSTS MATCHING A REGEXP, CREATING A NEW SUBSET:
1349 elif choice in menu[12][1]:
1350 search_term_regexp = input( message[19] )
1351 answer = input_qpost_key()
1352 tuples = search_qposts_regexp( search_term_regexp, key=answer )
1353 print( f'{tc}Found {bw} {len(tuples)} {tc} Qposts matching regexp="{search_term_regexp}".{ts}' )
1354 _QMAP_IDS = handle_search_results( tuples, answer )
1355 #(12) DATE SEARCH IN ALL QPOSTS, CREATING A NEW SUBSET:
1356 elif choice in menu[13][1]:
1357 date_condition = input( message[9] )
1358 answer = input_qpost_key()
1359 tuples = search_qposts_by_date( date_condition )
1360 print( f'{tc}Found {bw} {len(tuples)} {tc} Qposts matching date condition="{date_condition}".{ts}' )
1361 _QMAP_IDS = handle_search_results( tuples, answer )
1362 #(13) SEARCH Q-CLOCK ALIGNED QPOSTS FOR DATE, CREATING A NEW SUBSET:
1363 elif choice in menu[14][1]:
1364 date_string = input( message[13] )
1365 if not date_string: date_string = 'today'
1366 tuples = qclock_get_aligned_qposts_for_date( date_string )
1367 print( f'{tc}Found {bw} {len(tuples)} {tc} aligned Qposts for date={date_string}.{ts}' )
1368 _QMAP_IDS = handle_search_results( tuples, '', False )
1369 #(14) SEARCH Q-CLOCK ALIGNED QPOSTS FOR QMAP ID, CREATING A NEW SUBSET:
1370 elif choice in menu[15][1]:
1371 qmap_id = input( message[14] )
1372 if qmap_id.isdigit() and is_valid_qmap_id( int( qmap_id ) ): qmap_id = int( qmap_id )
1373 else: qmap_id = n_posts
1374 tuples = qclock_get_aligned_qposts_for_qmap_id( qmap_id )
1375 print( f'{tc}Found {bw} {len(tuples)} {tc} aligned Qposts for Qmap ID={qmap_id}.{ts}' )
1376 _QMAP_IDS = handle_search_results( tuples, '', False )
1377 #(15) SEARCH Q-CLOCK ALIGNED QPOSTS FOR CLOCKTIME, CREATING A NEW SUBSET:
1378 elif choice in menu[16][1]:
1379 s_time = input( message[16] )
1380 s_parts = s_time.split()
1381 if len( s_parts ) < 2: s_parts = s_time.split( ':' )
1382 if len( s_parts ) < 2: s_parts = s_time.split( ',' )
1383 try: qtime = ( int( s_parts[0] ), int( s_parts[1] ) )
1384 except: qtime = (4,20)
1385 # normalize clocktime.
1386 qtime = ( ( qtime[0] + qtime[1] // 60 ) % 12, qtime[1] % 60 )
1387 tuples = qclock_get_aligned_qposts_for_clocktime( qtime, _QCLOCK_ROUND_NEAREST )
1388 hands_angle = clocktime_to_angle( qtime[0], qtime[1] )
1389 hr_pos = clocktime_hourhand_pos( qtime[0], qtime[1] )
1390 hr_pos_rounded = f'{[int,round][_QCLOCK_ROUND_NEAREST](hr_pos)}'
1391 hr_pos_info = f'' if hr_pos == float( hr_pos_rounded ) else f' (rounded off from {hr_pos:0.2f})'
1392 sqtime = f'{qtime[0]:02d}:{qtime[1]:02d}'
1393 print( message[17].format( sqtime, hr_pos_rounded, hr_pos_info, f'{hands_angle:0.2f}' ) )
1394 print( f'{tc}Found {bw} {len(tuples)} {tc} aligned Qposts for Qclocktime {bw} ({sqtime}) {tc}.{ts}' )
1395 _QMAP_IDS = handle_search_results( tuples, '', False )
1396 #(16) CHECK FOR NEW QPOSTS:
1397 elif choice in menu[18][1]:
1398 n_posts = check_update_local_qposts()
1399 #(17) DOWNLOAD QPOSTS JSON-FILE FROM QANON.PUB:
1400 elif choice in menu[19][1]:
1401 qposts = download_qposts_json( _URL_QPOSTS )
1402 if qposts:
1403 _QPOSTS = qposts; n_posts = len(_QPOSTS)
1404 print( f'{tc}File "Qposts.json" with {bw} {n_posts} {tc} Qposts successfully downloaded and saved to your Downloads folder.{ts}' )
1405 #(18) EXPORT QPOSTS TO SQLITE3 DATABASE:
1406 elif choice in menu[20][1]:
1407 qposts_to_sqlite( _QPOSTS, _URL_QPOSTS_DB )
1408 #(19) DOWNLOAD IMAGES FROM THE CURRENT SUBSET:
1409 elif choice in menu[21][1]:
1410 answer = input( message[21] )
1411 download_qpost_images( _QMAP_IDS, answer.lower() not in ['n','no'] )
1412 #(20) PERFORM SQL SELECT QUERY IN QPOST.DB:
1413 elif choice in menu[23][1]:
1414 sql = input( message[22] )
1415 if sql.lower().startswith( 'select ' ):
1416 if not sql.endswith( ';' ): sql += ';'
1417 res = qposts_sqlite_query( _URL_QPOSTS_DB, sql )
1418 print( f'{tc}SQL Query Result:{ts} ', res )
1419 else: print( message[23] )
1420 else: pass
1421 ##### END of qposts_terminal_loop()
1422
1423
1424#################
1425# Auxiliary Methods:
1426
1427def input_menu( menu, header='', prompt='> ', invalid='\033[1;47;31mInvalid choice.\033[0m', quits=['q','quit','exit'], visible={} ):
1428 '''Ask user to input a choice from a menu.
1429 Menuitems can be displayed in groups, which can be individually collapsed or expanded by entering the group key.
1430 To collapse or expand all groups at once, the user can enter the builtin commands HIDE or SHOW respectively.
1431 NB. Menuitems that are currently hidden, are not valid choices; First the menuitem must be made visible before it can be chosen.
1432 The function returns the key of the chosen menuitem, or '_EXIT' if the user chose to quit.
1433 <menu> : An ordered list of tuples, each with either 2 or 3 elements:
1434 For a group-header item, pass a 2-tuple( str, str ) where the first element is the header text to be displayed
1435 (including its key), and the second element is a key which identifies a group of menuitems.
1436 For a choosable menu item, pass a 3-tuple( str, list, str ) where the first element is the displayed text for this menuitem
1437 (including its preferred key), the 2nd element is a list of keys that the user can enter to select this particular choice,
1438 and the 3rd element is the key of the group that this menuitem belongs to ( i.e. the 2nd element of a group-header item ).
1439 <header> : string to be displayed before the list of menu choices; Leave empty if you don't want a menu header to be displayed.
1440 <prompt> : string to be displayed after the list of menu choices; This string should be asking for user input.
1441 <invalid> : string to be displayed when the user input is not recognized.
1442 <quits> : list of lowercase string commands that will exit the input loop (when typed in any case).
1443 <visible> : dict of {group:int} defining the initial visibility for each group; Updated in-place when user changes settings.'''
1444 # Commands to collapse all menu groups.
1445 _COLLAPSE = ['hide', 'collapse', 'none']
1446 # Commands to expand all menu groups.
1447 _EXPAND = ['show', 'expand', 'all']
1448 def print_menu():
1449 m_keys = []
1450 if header: print( header )
1451 # print visible menuitems and construct a list of valid keys:
1452 for item in menu:
1453 if isinstance( item, tuple ) and len( item ) >= 2:
1454 if len( item ) == 2: print( item[0] )
1455 elif visible.get( item[2], 1 ): m_keys.extend( item[1] ); print( item[0] )
1456 return m_keys
1457 ch, menuitem_keys, group_keys = '', [], []
1458 for item in menu: # Populate visibility dict and list of group keys:
1459 if isinstance( item, tuple ) and len( item ) == 2:
1460 if isinstance( item[1], str ):
1461 if item[1] not in visible: visible[ item[1] ] = 1
1462 group_keys.append( item[1] )
1463 menuitem_keys = print_menu() # Display the menu:
1464 while ch not in menuitem_keys:
1465 ch = input( prompt )
1466 if ch.lower() in quits: return '_EXIT' # User chose to Quit.
1467 if ch.lower() in _COLLAPSE: # Hide all menu groups.
1468 for gv in visible: visible[ gv ] = 0
1469 menuitem_keys = print_menu()
1470 elif ch.lower() in _EXPAND: # Show all menu groups.
1471 for gv in visible: visible[ gv ] = 1
1472 menuitem_keys = print_menu()
1473 elif ch in group_keys: # Hide/Show a specific menu group.
1474 visible[ ch ] = 1 - visible[ ch ]
1475 menuitem_keys = print_menu()
1476 elif ch in menuitem_keys: return ch # User chose a menu option.
1477 elif invalid: print( invalid )
1478 return '_ERROR'
1479
1480
1481def ansi_hyperlink( uri, text='Ctrl-Click Here', fg=None, bg=None, style=None ):
1482 '''Returns a string that can be displayed as a Ctrl-clickable hyperlink in the terminal.
1483 User can also right-click on the link to popup a contextmenu with options 'Open Hyperlink' and 'Copy Hyperlink Address'.
1484 Hovering the mouse over the hyperlink will popup a Tooltip showing the target uri.
1485 Hyperlink targets are opened using the system's default application for the target type.
1486 <uri> should be an urlencoded string containing only ascii characters 32 to 126, starting with an uri scheme identifier.
1487 Supported uri schemes a.o.: "http://", "https://", "ftp://", "file://", "mailto:".
1488 <text>: String representing the Ctrl-clickable text to be displayed.
1489 <fg>: None, or a 3-tuple of integers representing the RGB-values of the Foreground Color for <text>.
1490 <bg>: None, or a 3-tuple of integers representing the RGB-values of the Background Color for <text>.
1491 <style>: None, or a 5-tuple of Booleans for displaying <text> in Bold, Italic, Underline, Strikethrough, Blink.
1492 NB. if you pass <fg>, <bg>, and/or <style>, then any existing formatting of the text before the link will not be continued
1493 after the link. In that case, pass non-destructive ansi-code via <text> itself.'''
1494 if not (fg or bg or style): return fr"]8;;{uri}\\{text}]8;;\\"
1495 scv = [('','1;'),('','3;'),('','4;'),('','9;'),('','6;')]
1496 sbg = f"48;2;{bg[0]};{bg[1]};{bg[2]}" if bg and len( bg ) >= 3 else ''
1497 sfg = f"{';' if sbg else ''}38;2;{fg[0]};{fg[1]};{fg[2]}" if fg and len( fg ) >= 3 else ''
1498 stl = ''.join( [scv[i][bool(s)] for i,s in enumerate( style )] ) if style and len( style ) == 5 else ''
1499 stl = stl if sbg or sfg else stl[:-1]
1500 rst = '[0m' if stl or sbg or sfg else ''
1501 return f"]8;;{uri}\\[{stl}{sbg}{sfg}m{text}{rst}]8;;\\"
1502
1503
1504def collapse_user( str_path ): # inverse of os.path.expanduser()
1505 '''Replaces the user directory in a path by a tilde ( to hide the user name ).'''
1506 return str_path.replace( os.path.expanduser('~'), '~' )
1507
1508
1509def download_binary_file( url, save_url ):
1510 '''Tries to download a file from the Internet, and save it into the specified location <save_url>.
1511 Does NOT check if the file type indicated in <save_url> is the same as the file type from <url>.
1512 This function returns True if the download succeeded, else it returns False.'''
1513 def printq( msg ): print( msg ); return False
1514 response, headers = None, {'User-Agent': 'Mozilla/5.0'}
1515 try: response = urlopen( Request( url, headers=headers) ) # from urllib.request import Request, urlopen
1516 except URLError as e: # from urllib.error import URLError
1517 return False; printq( f"{er}URLError; cannot download file '{url}.'\tReason: {e.reason}.{ts}" )
1518 except: printq( f"{er}Error downloading file '{url}'.{ts}" )
1519 if not response: return False
1520 elif response.status != 200:
1521 printq( f"{er}Download Failure: Response has status code {response.status}.{ts}" )
1522 else:
1523 try:
1524 with open( save_url, 'wb' ) as f: f.write( response.read() )
1525 return True
1526 except: pass # ConnectionResetError
1527 return False
1528
1529
1530def parse_datetime_string( datetime_string='now', format_as='%c', parserinfo=None ):
1531 '''Parses a datetime string such as "Sept 17th, 1984 at 01:30 AM", and returns a 3-tuple containing the parsed datetime,
1532 a string expressing the parsed datetime in the specified format, and a rest tuple of unparsed tokens.
1533 If the parsing failed, this function returns (None,'','').
1534 <datetime_string>: String specifying a datetime to be parsed; The string can also specify a timestamp.
1535 <format_as>: determines the format of the datetime string to be returned; Default "%c" is locale date&time format.
1536 if the format is '' or None, a floating point timestamp will be returned instead of a string.
1537 NB. Uses the module dateutil; results are not so good for verbose date strings inside a sentence.'''
1538
1539 #import dateutil.parser as dateparser
1540 # #import datetime
1541 dts = datetime_string.strip() if isinstance( datetime_string, str ) else 'now'
1542 if dts.lower() in [ 'now', 'today' ]: dt, rest = datetime.datetime.now(), ()
1543 elif is_numeral( dts ): dt, rest = datetime.datetime.fromtimestamp( float( dts ), tz=None ), () # interpret input as timestamp.
1544 else:
1545 try: dt, rest = dateparser.parse(
1546 dts,
1547 parserinfo=parserinfo,
1548 default=None,
1549 dayfirst=True,
1550 yearfirst=False,
1551 ignoretz=True,
1552 fuzzy_with_tokens=True
1553 )
1554 # input could not be parsed. (Pass a custom parserinfo for the local user language?).
1555 except ValueError: return (None,'','')
1556 # OverflowError?: parsed date exceeds the largest valid C integer.
1557 except: return (None,'','')
1558 if dt: return dt, dt.strftime( format_as ) if format_as else dt.timestamp(), rest
1559 return (None,'','')
1560
1561
1562def seconds_to_timestring( seconds, units=['w','d','h','m','s'], add_s=False, separator=' ' ):
1563 '''Turns a number of seconds into a human-readable duration string, expressed in weeks, days, hours, minutes, and seconds.
1564 <seconds>: The total number of seconds in the duration; Can pass a float, but the decimal part is not used.
1565 <units> : Symbols for each of the 5 durations (week, day, hour, minute, second), or <None> to use the verbose English words.
1566 <add_s> : If True, the character 's' will be appended after the unit symbol, if the number for that unit is larger than 1.
1567 <separator>: String to put in between each of the number/unit pairs.
1568 Adapted from function elapsed_time(): http://snipplr.com/view/5713/python-elapsedtime-human-readable-time-span-given-total-seconds/'''
1569 assert( isinstance( seconds, (int,float) ) )
1570 if not units or len(units) < 5: units = [' week',' day',' hour',' minute',' second'] # NB. space before the units.
1571 if seconds == 0: return '%s%s' % ( '0', units[-1] + ( '', 's' )[add_s] )
1572 if seconds < 0: return '-' + seconds_to_timestring( -seconds, units, add_s, separator )
1573 duration, lengths = [], [ 604800, 86400, 3600, 60, 1 ]
1574 for unit, length in zip( units, lengths ):
1575 value = seconds // length
1576 if value >= 1:
1577 seconds %= length
1578 duration.append( '%s%s' % ( str(value), (unit, (unit, unit + 's')[value > 1])[add_s] ) )
1579 if seconds < 1: break
1580 return separator.join( duration )
1581
1582
1583def seconds_to_HMS( seconds, microseconds=False, days=True ):
1584 '''Converts <seconds> into a string of the format "HH:MM:SS" with optional microseconds and/or days.'''
1585 assert( isinstance( seconds, (int,float) ) and isinstance( microseconds, bool ) and isinstance( days, bool ) )
1586 if seconds == 0: return '00:00:00' + ('.000000' if microseconds else '')
1587 if seconds < 0: return '-' + seconds_to_HMS( -seconds )
1588 minutes, seconds = divmod( seconds, 60 )
1589 hours, minutes = divmod( minutes, 60 )
1590 msecs = f'{seconds:09.6f}' if microseconds else f'{int(seconds):02d}'
1591 if not days: return f'{int(hours):02d}:{int(minutes):02d}:{msecs}'
1592 ds, hours = divmod( hours, 24 )
1593 d = f"{ds} day{['s',''][bool(ds==1)]}, " if ds else ''
1594 return f'{d}{int(hours):02d}:{int(minutes):02d}:{msecs}'
1595
1596
1597def timestamp_day_bounds( timestamp ):
1598 '''Returns a 2-tuple with timestamps for the start & end of the day in which the specified <timestamp> falls.
1599 NB. The starting bound is Inclusive, the ending bound is Exclusive ( being the start of the next day ).'''
1600 ts_ordinal = datetime.datetime.fromtimestamp( timestamp, tz=None ).toordinal() # Serial Day Number.
1601 ts_start = datetime.datetime.fromordinal( ts_ordinal ).timestamp() # Timestamp Day-Start.
1602 return ts_start, ts_start + 86400
1603
1604
1605def clocktime_to_angle( hour, minute ):
1606 '''''Returns a float representing the angle between the hour and minute hands of an analog clock showing the specified time.'''
1607 H = int( hour ) % 12
1608 M = int( minute ) % 60
1609 return H * 30 - M * 6 + M / 2
1610
1611
1612def clocktime_hourhand_pos( hour, minute ):
1613 '''''Returns a float representing the position (0-59.916666) of the Hour-hand of an analog clock showing the specified time.'''
1614 H = int( hour ) % 12
1615 M = int( minute ) % 60
1616 return ( H + M / 60 ) * 5
1617
1618
1619def integer_list_to_range_string( integer_list, sep='-' ):
1620 '''Returns a string representation of the specified list of integers <integer_list>, where
1621 ranges of consecutive integers are compacted to only their start- and end, separated by <sep>.
1622 E.g. for input = [1,2,3,4,5,6,7,8,55] it returns the string "1-8,55".'''
1623 parts, previous, direction = [], None, 0
1624 for n in integer_list:
1625 if isinstance( n, int ) and previous is not None:
1626 if direction == 0:
1627 start = previous
1628 if n - previous == 1: direction = 1
1629 elif n - previous == -1: direction = -1
1630 else: parts.append( str( previous ) )
1631 elif n - previous != direction:
1632 direction = 0
1633 parts.append( str( start ) + sep + str( previous ) )
1634 previous = n
1635 parts.append( str( previous ) if direction == 0 else str( start ) + sep + str( previous ) )
1636 return ','.join( parts )
1637
1638
1639def range_string_to_integer_list( range_string, sep='-', validate=None, msgs=[] ):
1640 '''Returns a list of integers based on the specified <range_string>.
1641 <range_string>: Comma-separated string of: numbers and/or <sep>-separated number ranges.
1642 <sep>: Symbol separating the start and end of the number ranges inside <range_string>.
1643 <validate>: None, or Pass a validation function that should accept an integer and return a Boolean.
1644 <msgs>: List of 3 error messages in case the input is: Not a Number, Invalid Number, Invalid Range.'''
1645 def print_msg( i ):
1646 if msgs and len(msgs) > i: print( msgs[i] )
1647 intlist, items = [], range_string.split( ',' )
1648 for item in items:
1649 if sep in item: # separator: defines a range.
1650 range_ext = item.split( sep )[0:2]
1651 if range_ext[0].isdigit() and range_ext[1].isdigit():
1652 nm1, nm2 = int( range_ext[0] ), int( range_ext[1] )
1653 if not callable( validate ) or ( validate( nm1 ) and validate( nm2 ) ):
1654 subrange = list( range( nm1, nm2 - 1, -1 ) if nm2 < nm1 else range( nm1, nm2 + 1 ) )
1655 intlist.extend( subrange )
1656 else: print_msg( 1 ); break
1657 else: print_msg( 2 ); break
1658 elif item.isdigit():
1659 nm = int( item )
1660 if not callable( validate ) or validate( nm ): intlist.append( nm )
1661 else: print_msg( 1 ); break
1662 else: print_msg( 0 ); break
1663 return intlist
1664
1665
1666def description_to_range_string( desc, n_max, cur=[] ):
1667 '''Interpret commands: "all", "first N", "last N", "previous N", "next N", and asterisk shortcut: "*",
1668 into a range_string that can be converted by range_string_to_integer_list(). '''
1669 lcase, commands = desc.lower(), [ 'first', 'last', 'next', 'prev', 'previous', ]
1670 if lcase == 'all': return f'1-{n_max}'
1671 if any( lcase.startswith( cmd ) for cmd in commands ):
1672 parts = desc.split()
1673 amount = 1 if len(parts) < 2 or not parts[1].isdigit() else int( parts[1] )
1674 amount = min( max( 1, amount ), n_max )
1675 if lcase.startswith( 'first' ): return f'1-{amount}'
1676 if lcase.startswith( 'last' ): return f'{n_max}-{n_max-amount+1}'
1677 cmin, cmax = ( min( cur ), max( cur ) ) if cur else ( 1, n_max )
1678 if lcase.startswith( 'next' ): return f'{min( max( 1, cmax + 1 ), n_max )}-{min( max( 1, cmax + amount ), n_max )}'
1679 if lcase.startswith( 'prev' ): return f'{min( max( 1, cmin - 1 ), n_max )}-{min( max( 1, cmin - amount ), n_max )}'
1680 else: return desc.replace( '*', f'{n_max}' ) # asterisk * means the maximum value <n_max>.
1681
1682
1683def count_frequencies( text, lex=0 ):
1684 '''Returns a list of 2-tuples(str, int) containing the count of each lexical element in <text>.
1685 <lex>: Determines which lexical element to count: 0=count characters; 1=count words.'''
1686 # from collections import Counter
1687 if isinstance( text, ( list, tuple, dict ) ): text = str( text )
1688 if isinstance( text, str ):
1689 if lex == 0: return Counter( text ).most_common()
1690 elif lex == 1: return Counter( text.split() ).most_common()
1691 return []
1692
1693
1694def longest_common_substring( data ):
1695 ''' Finds the longest common substring from a list of strings.'''
1696 # From https://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python/2894073#2894073
1697 substr = ''
1698 if len( data ) == 1: return data[0] # only 1 element: return itself as longest string.
1699 if len( data ) > 1:
1700 d, n = data[0], len( data[0] )
1701 if n > 0:
1702 for i in range( n ):
1703 for j in range( n - i + 1 ):
1704 if j > len( substr ) and is_common_substring( d[i:i+j], data ):
1705 substr = d[i:i+j]
1706 return substr
1707
1708def is_common_substring( find, data ):
1709 '''Used by longest_common_substring().'''
1710 if len( data ) < 1 or len( find ) < 1: return False
1711 for i in range( len(data) ):
1712 if find not in data[i]: return False
1713 return True
1714
1715
1716def is_numeral( s ):
1717 '''Returns True if the input string <s> represents either an integer number (e.g. '2500'), a floating
1718 point number (e.g. '2500.0'), a number expressed in scientific notation (e.g. '2.5E3'), or 'NaN'.'''
1719 try: _ = float( s ); return True
1720 except : return False
1721
1722
1723def f_all( s_list, s_before='', s_after='', sep=', ' ):
1724 return sep.join( [ s_before + k + s_after for k in s_list ] ) if s_list else ''
1725
1726
1727def split_list( lst ):
1728 '''Returns two "flat" lists: the first containing all items from the first dimension of <lst>,
1729 the second containing all items from the second and further dimensions of <lst>.'''
1730 def flatten_list( lst, newlist=[] ):
1731 for item in lst:
1732 if isinstance( item, list ): flatten_list( item, newlist )
1733 else: newlist.append( item )
1734 dim_one, dim_rest = [], []
1735 for item in lst:
1736 if isinstance( item, list ): flatten_list( item, dim_rest )
1737 else: dim_one.append( item)
1738 return dim_one, dim_rest
1739
1740
1741#################
1742# Class Simple_Logic_Expression
1743
1744class Simple_Logic_Expression:
1745 '''Represents a simple logical expression such as 'A and B or not C'.
1746 Only supports the logical connectives And, Or, Not, and parentheses;
1747 Atoms containing spaces or parentheses should be enclosed in "(double) quotation marks".'''
1748
1749 def __init__( self, str_expression, parse_format=1, eval_func=[], eval_args=(), operators=[] ):
1750 '''<str_expression>: String containing the simple logic expression to be parsed, e.g: 'A and B or not C'.
1751 <operators> : List of 5 symbols for [And, Or, Not, Left Paren, Right Paren], that can be used in <str_expression>.
1752 <parse_format> : Determines the format of the parsed output:
1753 0=list (prefix) e.g: ['or', ['and', 'A', 'B'], ['not', 'C']];
1754 1=string (infix) e.g: "(A and B) or (not C)".
1755 <eval_func> : pass an evaluation function that returns a Boolean for each atom used in <str_expression>,
1756 or pass a list of atoms that are used in <str_expression>, and whose value is <True>.
1757 <eval_args> : optional tuple of arguments to pass on to the evaluation function <eval_func>.'''
1758 if not operators or len( operators ) < 5: operators = [ 'and', 'or', 'not', '(', ')' ]
1759 self.expression = str_expression
1760 self.eval_func = eval_func
1761 self.eval_args = eval_args
1762 self.format = min(max( 0, parse_format ), 1) # 0=list (prefix); 1=string (infix).
1763 self._OP_AND = operators[0] #'and'
1764 self._OP_OR = operators[1] #'or'
1765 self._OP_NOT = operators[2] #'not'
1766 self._PAR_L = operators[3] #'('
1767 self._PAR_R = operators[4] #')'
1768
1769 def parse( self, parse_format=None ):
1770 '''Parse the current expression, optionally overriding the current parse format.
1771 based on: https://www.howtobuildsoftware.com/index.php/how-do/gbu/string-algorithm-parsing-algorithm-
1772 to-add-implied-parentheses-in-boolean-expression'''
1773
1774 def stream_starts_with( stream, token ):
1775 return stream[0:len(token)] == list(token)
1776
1777 def pop( stream, token ):
1778 if stream_starts_with( stream, token ):
1779 del stream[0:len(token)]
1780 return True
1781 return False
1782
1783 def parse_primary( stream ):
1784 if pop( stream, '"' ): return parse_enclosure( stream, '"', '"' ) # parse double quote.
1785 while pop( stream, ' ' ): pass
1786 if pop( stream, self._PAR_L ):
1787 e = parse_or( stream )
1788 pop( stream, self._PAR_R )
1789 return e
1790 return parse_atom( stream )
1791
1792 def parse_enclosure( stream, enc_left, enc_right ):
1793 r = [ '', enc_left][self.format] # keep/restore enclosure symbols if format=1.
1794 while stream and not pop( stream, enc_right ):
1795 r += stream.pop(0)
1796 while pop( stream, ' ' ): pass
1797 return r + [ '', enc_right][self.format]
1798
1799 def parse_binary( stream, operator, func ):
1800 while pop( stream, ' ' ): pass
1801 es = [func( stream )]
1802 while pop( stream, operator ): es.append( func( stream ) )
1803 if self.format == 0: return [operator, *es] if len(es) > 1 else es[0]
1804 else: return self._PAR_L + ' {} '.format(operator).join(es) + self._PAR_R if len(es) > 1 else es[0]
1805
1806 def parse_unary( stream ):
1807 while pop( stream, ' ' ): pass
1808 if pop( stream, self._OP_NOT ):
1809 if self.format == 0: return [ self._OP_NOT, parse_unary( stream ) ]
1810 else: return f'({self._OP_NOT} {parse_unary( stream )})'
1811 return parse_primary( stream )
1812
1813 def parse_or( stream ):
1814 while pop( stream, ' ' ): pass
1815 p = parse_binary( stream, self._OP_OR, parse_and )
1816 return p if p else parse_unary( stream )
1817
1818 def parse_and( stream ):
1819 while pop( stream, ' ' ): pass
1820 p = parse_binary( stream, self._OP_AND, parse_unary )
1821 return p if p else parse_unary( stream )
1822
1823 def parse_atom( stream ):
1824 atom = ''
1825 while stream and not pop( stream, ' ' ) and not stream_starts_with( stream, self._PAR_R ):
1826 atom += stream.pop(0)
1827 return atom
1828
1829 if parse_format is not None: self.format = parse_format
1830 #if not isinstance( self.expression, (list,str) ): return self.expression
1831 output = parse_or( list( self.expression ) )
1832 return output if self.format == 0 else output[1:-1] # Removes the outermost pair of parentheses.
1833
1834
1835 def evaluate( self, eval_func=None, eval_args=None ):
1836 '''Evaluate the current expression, optionally overriding the current atom evaluation function (or list).'''
1837
1838 def evaluate_atom( atom ):
1839 if isinstance( self.eval_func, list ): return atom in self.eval_func
1840 if callable( self.eval_func ): return self.eval_func( atom, *self.eval_args )
1841
1842 def evaluate_op( op_list ):
1843 operator = op_list[0]
1844 truthval = evaluate_output( op_list[1] )
1845 if operator == self._OP_NOT: return not truthval
1846 if operator == self._OP_AND:
1847 for arg in op_list[2:]: truthval = truthval and evaluate_output( arg )
1848 return truthval
1849 if operator == self._OP_OR:
1850 for arg in op_list[2:]: truthval = truthval or evaluate_output( arg )
1851 return truthval
1852 return truthval
1853
1854 def evaluate_output( output ):
1855 if isinstance( output, list ): return evaluate_op( output )
1856 if isinstance( output, str ): return evaluate_atom( output )
1857 return output
1858
1859 if eval_func is not None: self.eval_func = eval_func
1860 if eval_args is not None: self.eval_args = eval_args
1861 if not isinstance( self.expression, (list,str) ): return evaluate_atom( self.expression )
1862 output = self.parse( parse_format=0 ) # must be parse_format=0 (=Prefix list).
1863 return evaluate_output( output )
1864
1865
1866 def find_in( self, text ):
1867 '''Interprets the current expression as a complex search-string, and checks if it matches the target <text>.'''
1868 def eval_atom( atom ): return atom in text
1869 return self.evaluate( eval_atom )
1870
1871
1872 def match_value( self, value, eval_func ):
1873 '''<eval_func>: required evaluation function taking at least 2 arguments: the current atom and <value>.'''
1874 return self.evaluate( eval_func, (value,) )
1875
1876# End of Simple_Logic_Expression
1877
1878
1879def evaluate_comparison( value1, op, value2 ):
1880 if op.lower() in ['on']: # Returns True if value2 is a timestamp falling on the same calendar day as value1.
1881 ts_start, ts_end = timestamp_day_bounds( value2 )
1882 return value1 >= ts_start and value1 < ts_end
1883 elif op in ['>=']: return value1 >= value2
1884 elif op in ['<=']: return value1 <= value2
1885 elif op in ['!=']: return value1 != value2
1886 elif op in ['==', '=']: return value1 == value2
1887 elif op in ['>']: return value1 > value2
1888 elif op in ['<']: return value1 < value2
1889
1890
1891def evaluate_datetime_condition( dt_condition, timestamp ):
1892 '''Returns True if the expression "<timestamp><dt_condition>" evaluates as True.
1893 <dt_condition>: Numerical Timestamp, or a String of the format "COMP DATETIME" (including double quotation marks),
1894 where COMP is one of the comparison operators in [ 'on', '>=', '<=', '!=', '==', '=', '>', '<' ],
1895 and where DATETIME is a valid datetime expression, a timestamp, or 'now'.
1896 The 'on'-operator yields True iff the <timestamp> falls on the same day as the DATETIME specified in <dt_condition>.
1897 <timestamp>: Integer or Float to be compared against the DATETIME specified in <dt_condition>.'''
1898 if isinstance( dt_condition, (int,float) ): # interpret dt_condition as timestamp.
1899 return evaluate_comparison( timestamp, 'on', dt_condition )
1900 for op in [ 'on', '>=', '<=', '!=', '==', '=', '>', '<' ]:
1901 if dt_condition.startswith( op ):
1902 dt_string = dt_condition[len(op):].strip()
1903 if is_numeral( dt_string ): dts = float( dt_string ) # interpret dt_string as timestamp.
1904 else: dt, dts, rest = parse_datetime_string( dt_string, '' ) # slow; instead pass a timestamp for dt_condition.
1905 if dts: return evaluate_comparison( timestamp, op, dts )
1906 return False
1907
1908
1909def evaluate_numeric_condition( num_condition_string, num ):
1910 '''Returns True if the expression "<num><num_condition_string>" evaluates as True.
1911 <num_condition_string>: String of the format "COMP NUMBER" (including double quotation marks),
1912 where COMP is one of the comparison operators in [ '>=', '<=', '!=', '==', '=', '>', '<' ],
1913 and NUMBER is any number string that can be casted into a float, e.g.: '1', '2.2e3', etc.
1914 <num>: Integer or Float to be compared against the NUMBER specified in <num_condition_string>.'''
1915 for op in [ '>=', '<=', '!=', '==', '=', '>', '<' ]:
1916 if num_condition_string.startswith( op ):
1917 num_string = num_condition_string[len(op):]
1918 try: return evaluate_comparison( num, op, float( num_string ) )
1919 except: pass # Error if float() fails.
1920 return False
1921
1922
1923#################
1924# Main:
1925
1926if __name__ == '__main__':
1927
1928 # Read optional commandline parameters to override default settings:
1929 # Type "python3 qposts_research.py -h" to see a list of options.
1930 descr = 'Python script to facilitate research/analysis of Qposts.'
1931 epilog = 'Thank you for using %(prog)s ... WWG1WGA!'
1932 h_tips = 'Add this flag to hide tooltips for abbreviations.'
1933 h_wrap = 'Wrap width (in characters) for the Qposts Text field.'
1934 h_indn = 'Amount of indentation for nested references.'
1935 t_cols = ['black','red','green','yellow','blue','magenta','cyan','white']
1936 h_tcol = 'Label Color: ' + '; '.join([ f'{i}=[3{i}m{c}[0m' for i,c in enumerate(t_cols)]) + '.'
1937 h_menu = 'Initial visibility (0 or 1) for each menu group.'
1938 h_qset = 'Initial subset of Qmap IDs, e.g. "all" or "last 100".'
1939 h_date = 'Strftime format to display a short date string.'
1940 h_dtm = 'Strftime format to display a long date&time string.'
1941 h_vers = 'Show program version and exit.'
1942
1943 formatter = argparse.MetavarTypeHelpFormatter
1944 parser = argparse.ArgumentParser( description=descr, epilog=epilog, formatter_class=formatter )
1945 parser.add_argument( '-nt', '--notips', help=h_tips, default=(not _SHOW_ABBR_TOOLTIPS), action='store_true' )
1946 parser.add_argument( '-w', '--wrap', type=int, help=h_wrap, default=_MAX_WRAP )
1947 parser.add_argument( '-i', '--indent', type=int, help=h_indn, default=len( _LVL_INDENT ) )
1948 parser.add_argument( '-c', '--color', type=int, help=h_tcol, default=argparse.SUPPRESS )
1949 parser.add_argument( '-d', dest='date', type=str, help=h_date, default=_DT_FORMAT )
1950 parser.add_argument( '-dtm', dest='datetime', type=str, help=h_dtm, default=_DTM_FORMAT )
1951 parser.add_argument( '-m', dest='menu', type=int, help=h_menu, default=argparse.SUPPRESS, nargs='*' )
1952 parser.add_argument( '-s', dest='subset', type=str, help=h_qset, default=[_INITIAL_SUBSET], nargs='+' )
1953 parser.add_argument( '--version', action='version', help=h_vers, version='%(prog)s version 0.0.1b (20200117)' )
1954 args = parser.parse_args()
1955
1956 _INITIAL_SUBSET = ' '.join( args.subset )
1957 _SHOW_ABBR_TOOLTIPS = not args.notips
1958 _LVL_INDENT = ' ' * args.indent
1959 _MAX_WRAP = args.wrap
1960 _DT_FORMAT = args.date
1961 _DTM_FORMAT = args.datetime
1962 if hasattr( args, 'color' ):
1963 if args.color >= 0 and args.color <= 7:
1964 tc = f'[0;3{args.color}m'
1965 if hasattr( args, 'menu' ):
1966 mg_len = len( _VISIBLE_MENU_GROUPS )
1967 mg_keys = list( _VISIBLE_MENU_GROUPS.keys() )
1968 for i, vis in enumerate( args.menu ):
1969 if i < mg_len: _VISIBLE_MENU_GROUPS[mg_keys[i]] = int( bool( vis ) )
1970
1971
1972 print( f'{tc}{datetime.datetime.now().strftime( _DTM_FORMAT )}' )
1973
1974 _QPOSTS = open_qposts_json( _URL_QPOSTS ) # Open local Qposts.json file.
1975 if not _QPOSTS:
1976 print( f"{er}Local file Qposts.json not found:{ts} Attempting to download Qposts.json from qanon.pub ..." )
1977 _QPOSTS = download_qposts_json( _URL_QPOSTS )
1978
1979 if _QPOSTS:
1980 n_qposts = check_update_local_qposts() # Check for Latest Qposts online.
1981 print( f'{tc}Local Qposts.json: containing {bw} {n_qposts} {ts+tc} Qposts.{ts}' )
1982
1983 if os.path.exists( _URL_QPOSTS_DB ): # Check if SQLite3 QPosts.db exists:
1984 n_recs = qposts_sqlite_count_records( _URL_QPOSTS_DB )
1985 print( f'{tc}Local Qposts.db: containing {bw} {n_recs} {ts+tc} Qpost Records.{ts}' )
1986
1987 range_string = description_to_range_string( _INITIAL_SUBSET, n_qposts )
1988 _QMAP_IDS = range_string_to_integer_list( range_string ) # Create subset of Qmap_ID's.
1989 print( f'{tc+cu}Current subset of Qmap IDs:{ts}', integer_list_to_range_string( _QMAP_IDS ) )
1990
1991 qposts_terminal_loop() # Start terminal input loop.
1992 else: print( f"{er}No Qposts.json available:{tc} Exiting Program ...{ts}" )
1993
1994quit()
1995# End