· 6 years ago · Jan 17, 2020, 05:34 PM
1#!/usr/bin/env python3
2# qposts_research.py
3# version 0.0.1b (20200117)
4
5# Below is a set of Python methods to facilitate research/analysis of Qposts.
6# - needs Python (version 3.4+ );
7# - uses only standard Python libraries;
8# - tested only with Python 3.7.5 on Ubuntu 19.10
9# - dictionary of Qpost abbreviations included.
10#
11# To run the script, type "python3 qposts_research.py" in your terminal (at the correct folder-location).
12# With this Python script you can:
13# - Download all Qposts as a single JSON-file from qanon.pub.
14# - Define arbitrary subsets of Qposts and print them out in any order.
15# - Perform a complex Text or Date Search using "and, or, and not"-operators.
16# - Perform a Regular Expression Search or group matching of Qpost data.
17# - Find Qclock-aligned dates, together with the Qposts posted on those dates.
18# - Download images linked in Qposts (100% success rate), including images from references (93% success).
19# - Export your Qposts JSON-file to an SQLite3 Database and perform SQL SELECT-queries from the terminal.
20#
21# Relevant Quotes from Q:
22# Q59: "Combine all posts and analyze."
23# Q993: "Learn how to archive offline."
24# Q22: "Study and prepare."
25#
26# This is Open-Source Freeware/QAnonware;
27# Released As-Is; No Warranty; Use at Own Risk.
28# May God Bless Q-team & QAnons Worldwide.
29# WWG1WGA
30
31import os
32import re
33import json
34import html
35import sqlite3
36import argparse
37import datetime
38import dateutil.parser as dateparser
39import xml.etree.ElementTree as xml_tree
40from urllib.request import Request, urlopen
41from urllib.error import URLError
42from collections import Counter
43from textwrap import wrap
44
45_MAX_WRAP = 120 # maximum wrap width for the Qposts "Text" field.
46_LVL_INDENT = ' ' * 4 # indented space for displaying nested references.
47_INITIAL_SUBSET = '*-1' # Determines the initial subset of Qmap IDs (can be 'all', 'first N', 'last N', etc.).
48_SHOW_ABBR_TOOLTIPS = True # True = Show Tooltips for known abbreviations in the Qposts "Text" field.
49_QCLOCK_ROUND_NEAREST = True # True = Qclock Hourhand Round-off to Nearest Minute; False = Round to Floor.
50_VISIBLE_MENU_GROUPS = {'A':1,'B':1,'C':1,'D':1} # Determines the initial visibility of each menu group (0=hidden; 1=visible).
51 # (Group D is only active if the user has an SQLite3 Qposts.db at _URL_QPOSTS_DB.)
52
53_DT_FORMAT = '%a %d %b %y' # strftime format to display a short date string.
54_DT_FORMAT_L = '%A %d %B %Y' # strftime format to display a long date string.
55_DTM_FORMAT = '%A %d %B %Y %X %Z' # strftime format to display a long date&time string.
56
57ok = '[1;48;2;190;190;190;38;2;10;100;20m' # green color for Success.
58er = '[1;48;2;190;190;190;38;2;160;30;50m' # red color for Errors.
59mc = '[0;48;2;190;190;190;38;2;30;100;50m' # color for input Labels.
60bl = '[1;48;2;132;173;191;30m' # background light blue for Qclock dates with at least 1 Qpost.
61mf = '[0;38;2;150;190;10m' # SQLite DB link color.
62gr = '[38;2;180;180;180m' # color for greyed-out text.
63bw = '[1;37;40m' # color for values: Bold white on black.
64tc = '[0;33m' # yellow for Qpost Labels + program text.
65iv = '[7m' # invert
66cu = '[3m' # italic
67bd = '[1m' # bold
68ts = '[0m' # reset to normal
69
70
71_SAFE_URL_QPOSTS = '~/Downloads/Qposts.json' # location for the downloaded Qposts.json file.
72_SAFE_URL_QPOSTS_DB = '~/Downloads/Qposts.db' # location for the Qposts SQLite-database.
73_SAFE_URL_QPOSTS_IMAGES = '~/Downloads/Qposts_images/' # location for downloaded Qpost images.
74
75_URL_QPOSTS = os.path.expanduser( _SAFE_URL_QPOSTS ) # un-anonymized versions of the url paths above.
76_URL_QPOSTS_DB = os.path.expanduser( _SAFE_URL_QPOSTS_DB )
77_URL_QPOSTS_IMAGES = os.path.expanduser( _SAFE_URL_QPOSTS_IMAGES )
78
79_QPOSTS = []
80_QMAP_IDS = []
81_QPOST_KEYS = [ 'timestamp', 'text', 'media', 'references', 'name', 'trip', 'userId', 'link',
82 'source', 'threadId', 'id', 'title', 'subject', 'email', 'timestampDeletion' ]
83 # key timestampDeletion: used in 5 Qposts: [124,229,231,232,240] (all on 4plebs).
84 # title: used in 248 Qposts (on 4plebs/8chan_cbts); subject: used in all other Qposts (on 8ch/8kun).
85
86_QPOST_DB_COLS = [ 'qmap_id' ]
87_QPOST_DB_COLS.extend( _QPOST_KEYS )
88_QPOST_DB_COLS.__setitem__( 4, 'refs' ) # NB. 'references' is a reserved word in SQLite3.
89
90_DOC_STRFTIME = 'https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior'
91_DOC_SQLITE_FUNC = 'https://www.sqlite.org/lang_corefunc.html'
92_DOC_SQLITE_SLCT = 'https://www.sqlitetutorial.net/sqlite-select/'
93
94
95# Dictionary of Qpost Abbreviations (Non-exhaustive; Collected from various sources):
96# NB. this dictionary needs careful ordering of the items:
97# - Keys that occur entirely inside another item's Value, must be placed BEFORE all such items (else the link formatting gets mixed up).
98# - Keys that occur entirely inside another item's Key, must be placed AFTER all such items (else only the smaller Key will match).
99# In this way the smaller Keys will have their own tooltip inside the larger Key, and the larger Key will also have its own tooltip,
100# unless the larger Key consists entirely of smaller Keys...
101_QPOST_ABBR = {
102 'AUS': 'Australia',
103 'AZ': 'Arizona',
104 'CA': 'California',
105 'EU': 'European Union',
106 'FL': 'Florida',
107 'FR': 'France',
108 'GER': 'Germany',
109 'HI': 'Hawaii',
110 'HK': 'Hong Kong',
111 'HKG': 'Hong Kong',
112 'LV': 'Las Vegas',
113 'MX': 'Mexico',
114 'NK': 'North Korea',
115 'NY': 'New York',
116 'NYC': 'New York City',
117 'NZ': 'New Zealand',
118 'PAK': 'Pakistan',
119 'RUS': 'Russia',
120 'SA': 'Saudi Arabia',
121 'SK': 'South Korea',
122 'TN': 'Tennessee',
123 'TX': 'Texas',
124 'UK': 'United Kingdom',
125 'UN': 'United Nations',
126 'US': 'United States',
127 'USA': 'United States of America',
128 'UT': 'Utah',
129 'VA': 'Virginia',
130 'WASH': 'Washington',
131 'ATL': 'Atlanta Airport',
132 'IAD': 'Dulles International Airport (Washington)',
133 'PVG': 'Shanghai Pudong Airport',
134 'AF1': 'Air Force 1 (Presidential Airplane)',
135 'AG': 'Attorney General',
136 'AMB': 'Ambassador',
137 'CEO': 'Chief Executive Officer',
138 'CEOs': 'Chief Executive Officers',
139 'CIGIE': 'Council of Inspectors General on Integrity and Efficiency',
140 'DAG': 'Deputy Attorney General',
141 'IG': 'Inspector General',
142 'NY_AG': 'Attorney General of New York State',
143 'POTUS': 'President of the United States',
144 'SCJ': 'Supreme Court Justice',
145 'SD': 'State Department',
146 'SIG': 'Signal; Special Interest Group',
147 'SIT RM': 'Situation Room (White House)',
148 'VP': 'Vice President',
149 'WH': 'White House',
150 'DoD': 'Department of Defense',
151 'DoT': 'Department of Transportation',
152 'DOJ': 'Department Of Justice',
153 'DOE': 'Department Of Energy',
154 'DHS': 'Department of Homeland Security',
155 'DNC': 'Democratic National Committee',
156 'A\'s': 'Agencies',
157 'ABCs': 'Alphabet agencies (Acronyms of US governmental agencies)',
158 'AFIA': 'Air Force Intelligence Agency',
159 'AIA': 'Army Intelligence Agency',
160 'CIA': 'Central Intelligence Agency',
161 'C_A': 'CIA (Lacking Intelligence)',
162 'DARPA': 'Defense Advanced Research Projects Agency',
163 'DIA': 'Defense Intelligence Agency',
164 'DNI': 'Director of National Intelligence',
165 'ESC': 'Electronic Security Command (USAF)',
166 'FBI': 'Federal Bureau of Investigation',
167 'FISA': 'Foreign Intelligence Surveillance Act',
168 'FISC': 'Foreign Intelligence Surveillance Court (FISA Court)',
169 'FOIA': 'Freedom Of Information Act',
170 'ICE': 'U.S. Immigration and Customs Enforcement',
171 'INSCOM': 'United States Army Intelligence and Security Command',
172 'IRS': 'Internal Revenue Agency',
173 'ITAC': 'Intelligence and Threat Analysis Center (US Army)',
174 'NASA': 'National Aeronautics and Space Administration',
175 'NCTC': 'National Counter Terrorism Center',
176 'NG': 'National Guard',
177 'NOIC': 'Naval Operational Intelligence Center (US Navy)',
178 'NPIC': 'National Photographic Intelligence Center (CIA)',
179 'NSA': 'National Security Agency',
180 'OCMC': 'Overhead Collection Management Center (NSA)',
181 'OIG': 'Office of the Inspector General',
182 'TSA': 'Transportation Security Administration',
183 'SRD': 'Special Research Detachment (US Army)',
184 'SS': 'Secret Service',
185 'USSS': 'United States Secret Service',
186 'FVEY': 'Five Eyes: an intelligence alliance comprising USA, UK, CAN, AUS, NZ',
187 '5 Eyes': 'FVEY (an intelligence alliance comprising USA, UK, CAN, AUS, NZ)',
188 'GCHQ Bude': 'UK Government satellite ground station and eavesdropping centre',
189 'GCHQ': 'Government Communications Headquarters (UK)',
190 'MI5': 'Military Intelligence Section 5 (UK Security Service)',
191 'MI6': 'Military Intelligence Section 6 (UK Secret Intelligence Service)',
192 'SIS': 'UK Secret Intelligence Service',
193 'MOSSAD': 'Israeli Secret Intelligence Service',
194 'MOS': 'MOSSAD (Israeli Secret Intelligence Service)',
195 'MSM': 'Mainstream Media',
196 'ARM': 'Anti-Republican Media',
197 'AP': 'Associated Press',
198 'AURN': 'American Urban Radio Networks',
199 'ABC': 'American Broadcasting Company; Alphabet agencies',
200 'BBC': 'British Broadcasting Corporation',
201 'BUZZF': 'BuzzFeed',
202 'CBS': 'Columbia Broadcasting System',
203 'CNN': 'Cable News Network',
204 'CNBC': 'Consumer News and Business Channel',
205 'HuffPo': 'Huffington Post',
206 'LAT': 'Los Angeles Times',
207 'NBC': 'National Broadcasting Company',
208 'MSNBC': 'US TV network partnership between Microsoft and NBC',
209 'NPR': 'National Public Radio',
210 'NYT': 'New York Times',
211 'OANN': 'One America News Network',
212 'PBS': 'Public Broadcasting Service',
213 'WaPo': 'Washington Post',
214 'WAPO': 'Washington Post',
215 'WASHPOST': 'Washington Post',
216 'WSJ': 'Wall Street Journal',
217 'FB': 'Facebook',
218 'GOOG': 'Google',
219 'WL': 'WikiLeaks',
220 'YT': 'YouTube',
221 'JFK JR': 'John F. Kennedy Junior (son of President John F. Kennedy)',
222 'JFK': 'John Fitzgerald Kennedy (35th US President); Gen. John Francis Kelly',
223 'DJT': 'Donald John Trump (45th US President)',
224 'Flynn JR': 'Michael Flynn Junior (son of General Flynn)',
225 'GHWB': 'George Herbert Walker Bush (41st US President)',
226 'GWB': 'George W. Bush (43rd US President)',
227 'HRC': 'Hillary Rodham Clinton (Secretary of State in Obama\'s first term)',
228 'Huma': 'Huma Abedin (Personal Assistant to Hillary Clinton)',
229 'IA': 'Information Assurance',
230 'IC': 'Intelligence Community',
231 'SC': 'Supreme Court; Special Counsel; Sara Carter (investigative reporter)',
232 'Perkins Coie': 'DNC’s private law firm',
233 'Fusion GPS': 'phony-intel firm was paid $1,024,408 by HRC/Perkins-Coie for creating the Steel Dossier',
234 'Crowdstrike': 'CA-based Cyber-security company that falsely claimed that Russia had hacked the DNC servers',
235 '\n+++': 'House of Saud', # added \n for best result.
236 '\n++': 'Rothschild family',
237 '\n+': 'George Soros (Globalist billionaire)',
238 '[]': 'kill box (area of interest)',
239 '[R]': 'Renegade (Secret Service codename for Barack Obama); Rothschild (?)',
240 '[E]': 'Eagle (Secret Service codename for Bill Clinton)',
241 '[D]': 'Democrat; Democratic',
242 '[D]s': 'Democrats',
243 '1-800-273-8255': 'Phone number of the Veterans Crisis Line',
244 '11.11.18.': 'IP address range of US DoD Network Information Center',
245 '187': 'Police Code for homicide',
246 '212-397-2255': 'Phone number of the Clinton Global Initiative',
247 '302': 'FD-302 form used by the FBI for taking notes during an interview',
248 '404': 'HTTP response code indicating "Page Not Found"',
249 '4-10-20': 'letter values of initials DJT (Donald John Trump)',
250 '4,10,20': 'letter values of initials DJT (Donald John Trump)',
251 '4, 10, 20': 'letter values of initials DJT (Donald John Trump)',
252 '4ch': '4chan (message board previously used by Q)',
253 '5:5': 'Loud and Clear',
254 '7th Floor': '"Shadow Government" within the SD that regularly met on the 7th floor of the Harry S. Truman Building in DC',
255 '8ch': '8chan (message board previously used by Q)',
256 '@JACK': 'Jack Dorsey (CEO of Twitter)',
257 '@Jack': 'Jack Dorsey (CEO of Twitter)',
258 '@Snowden': 'Edward Snowden (CIA/NSA spy who leaked NSA documents)',
259 '@SNOWDEN': 'Edward Snowden (CIA/NSA spy who leaked NSA documents)',
260 'A Cooper': 'Anderson Cooper (CNN anchor, son of Gloria Vanderbilt)',
261 'ADM R': 'Admiral Michael S. Rogers (Director of the NSA)',
262 'Adm R': 'Admiral Michael S. Rogers (Director of the NSA)',
263 'AJ': 'Alex Jones (Radio show host linked to Mossad)',
264 'AL-Q': 'Al-Qaeda (Islamic terrorist group pursuing NATO\'s geostrategic goals)',
265 'AL': 'Al Franken (Sen. D-MN)',
266 'AM': 'Andrew McCabe (FBI Deputy Director); Ante Meridiem',
267 'Anon': 'anonymous person',
268 'ANTIFA': '"Anti-Fascists" (Soros backed fascists/domestic terrorists)',
269 'AS': 'Adam Schiff (Rep. D-CA); Antonin Scalia (Supreme Court Associate Justice)',
270 'ASF': 'American Special Forces (?); Administrative Support Facility (?)',
271 'AW': 'Anthony Weiner (convicted pedophile ex-husband of Huma Abedin)',
272 'AWAN': 'Imran Awan (DNC IT staffer who blackmailed House Members)',
273 'B2': 'Stealth bomber; Bill Barr (US Attorney General under Trump)',
274 'B/H C': 'Bill & Hillary Clinton',
275 'BARR': 'Bill Barr (US Attorney General under GHWB and Trump)',
276 'BB': 'Bill Barr (US Attorney General under GHWB and Trump)',
277 'BC': 'Bill Clinton (42nd US President)',
278 'BDT': 'Bulk Data Transfer; Blunt & Direct Time; Bangladeshi Taka (currency)',
279 'BGA': 'Bundesverband Großhandel, Außenhandel (German trade association)',
280 'BHO': 'Barack Hussein Obama (44th US President)',
281 'BIDEN': 'Joseph Biden (VP under Obama)',
282 'BO': 'Board Owner; Barack Obama; Bruce Ohr (Associate Deputy AG)',
283 'BOD': 'Board of Directors',
284 'BODs': 'Boards of Directors',
285 'BP': 'Border Patrol; Bill Priestap (FBI Dep. Dir. of Counterintelligence under Obama and Trump)',
286 'BRENNAN': 'John Brennan (23rd CIA Director)',
287 'BS': 'Bernie Sanders (Sen. I-VT)',
288 'CC': 'Chelsea Clinton (daughter of Bill & Hillary Clinton)',
289 'CF': 'Clinton Foundation',
290 'CFR': 'Council on Foreign Relations',
291 'CHAI': 'Clinton Health Access Initiative',
292 'C-Info': 'Confidential Information',
293 'CLAPPER': 'Director of National Intelligence under Obama',
294 'CLAS': 'Classification; Classified',
295 'CLAS_OP_IAD_': 'Classified Operation at Dulles International Airport (?)',
296 'Clowns In America': 'CIA',
297 'CLOWNS IN AMERICA': 'CIA',
298 'CM': 'CodeMonkey (8kun Admin); Cheryl Mills (Adviser to Hillary Clinton)',
299 'CoC': 'Chain of Command; Chain of Custody',
300 'COC': 'Chain Of Command; Chain Of Custody',
301 'COMEY': 'James Comey (7th FBI Director)',
302 'CORSI': 'Jerome Corsi, Mossad asset/agent',
303 'CoS': 'Chief of Staff',
304 'COV': 'Covert',
305 'COVFEFE': 'Communications Over Various Feeds Electronically For Engagement Act',
306 'CRUZ': 'Ted Cruz (Sen. R-TX)',
307 'CS': 'Chuck Schumer (Sen. D-NY); Christopher Steele (former MI6); Civil Service',
308 'D\'s': 'Democrats',
309 'D’s': 'Democrats',
310 'D+R+I': 'Democrat + Republican + Independent',
311 'D5': 'Highest avalanche rating; December 5th; Chess move; 45=Trump',
312 'DACA': 'Deferred Action for Childhood Arrivals (US immigration policy)',
313 'DC': 'District of Columbia (Washington); Dan Coats (DNI under Trump); Dick Cheney (VP under G.W.Bush)',
314 'DDoS': 'Directed Denial of Service (computer attack)',
315 'DECLAS': 'Declassification; Declassified',
316 'DEFCON': 'Defense Condition; Definitely Confirmed',
317 'DF': 'Dianne Feinstein (Sen. D-CA)',
318 'DL': 'Driver\'s License; David Laufman (Federal prosecutor); David Lawrence (Counsel to the Assistant AG)',
319 'DM': 'Denis McDonough (White House Chief of Staff under Obama)',
320 'DOA': 'Date Of Arrival; Dead Or Alive',
321 'Donna': 'Donna Brazille (Hillary staffer)',
322 'Dopey': 'Prince Al-Waleed bin Talal bin Abdulaziz al Saud',
323 'DS': 'Deep State',
324 'DWS': 'Debbie Wasserman Schulz (Rep. D-FL, DNC Chair under Obama)',
325 'Eagle': 'Secret Service codename for President Bill Clinton',
326 'Evergreen': 'Secret Service codename for Hillary Clinton',
327 'EBS': 'Emergency Broadcast System',
328 'EC': 'Eric Ciaramella (CIA agent)',
329 'EG': 'Evergreen (Secret Service codename for Hillary Clinton)',
330 'EH': 'Eric Holder (US Attorney General under Obama)',
331 'EM': 'Emergency; Elon Musk (CEO of SpaceX and Tesla Inc.)',
332 'EMP': 'Electromagnetic pulse',
333 'EMS': 'Emergency Medical Services; Emergency Medical System',
334 'EO': 'Executive Order',
335 'EOs': 'Executive Orders',
336 'EPSTEIN': 'Jeffrey Epstein (Billionaire who operated an elite pedophile ring for the Mossad)',
337 'ES': 'Eric Schmidt (CEO of Google); Edward Snowden (CIA double agent who leaked NSA secrets)',
338 'EST': 'Eastern Standard Time',
339 'F + D': 'Foreign and Domestic',
340 'F&F': 'Fast and Furious - Feinstein\'s failed gun sale attempt',
341 'F2F': 'Face to Face',
342 'f2f': 'face to face',
343 'F9': 'Message Authentication Code integrity algorithm used by Facebook',
344 'FED': 'Federal Reserve System (US Central Bank); Federal',
345 'FEINSTEIN': 'Dianne Feinstein (Sen. D-CA)',
346 'FF': 'False Flag',
347 'FG&C': 'For God And Country',
348 'FIRE & FURY': 'President Trump\'s warning to North Korea (8 Aug 2017)',
349 'FISA_T_SURV': 'Targeted Surveillance authorized under Section 702 of the FISA Amendments Act',
350 'FLYNN': 'Gen. Michael T. Flynn (National Security Advisor under Obama, fired for patriotism)',
351 'FY': 'Fiscal Year',
352 'G v E': 'Good versus Evil',
353 'GA': 'Great Awakening',
354 'GANG OF 8': 'Oversight board of the U.S. intelligence community',
355 'GANG OF EIGHT': 'Oversight board of the U.S. intelligence community',
356 'GDP': 'Gross Domestic Product',
357 'GINA': 'Gina Haspel (25th CIA Director)',
358 'GJ': 'Grand Jury',
359 'GOODLATTE': 'Bob Goodlatte (Rep. R-VA)',
360 'GOP': 'Grand Old Party (Republican Party)',
361 'gov\'t': 'Government',
362 'govt': 'Government',
363 'Gov’t': 'Government',
364 'Gov': 'Governor; Government',
365 'GOV': 'Government',
366 'GOWDY': 'Trey Gowdy (Rep. D-SC)',
367 'GPS': 'Global Positioning System',
368 'GRASSLEY': 'Chuck Grassley (Sen. R-IA)',
369 'GS': 'George Soros (Billionaire globalist investor)',
370 'GZ': 'Ground Zero',
371 'HA': 'Huma Abedin (Personal Assistant to Hillary Clinton)',
372 'HAM radio': 'Amateur radio',
373 'HEC': 'House Ethics Committee',
374 'HLS': 'Harvard Law School',
375 'HOLDER': 'Eric Holder (US Attorney General under Obama)',
376 'HOROWITZ': 'Michael Horowitz (DOJ Inspector General)',
377 'HS': 'Homeland Security',
378 'H-relief': 'Haiti earthquake relief effort coordinated by Bill Clinton',
379 'HUBER': 'John Huber (US Attorney for Utah)',
380 'HUMA': 'Harvard University Muslim Alumni; Huma Abedin',
381 'HUMINT': 'Human Intelligence',
382 'HUNTER': 'Hunter Biden (son of Joe Biden)',
383 'HUSSEIN': 'Barack Hussein Obama (44th US President)',
384 'HW': 'Hollywood',
385 'HWOOD': 'Hollywood',
386 'H wood': 'Hollywood',
387 'H-wood': 'Hollywood',
388 'ICBM': 'Inter-Continental Ballistic Missile',
389 'ICIG': 'Inspector General of the Intelligence Community',
390 'ISIS': 'Israeli Secret Intelligence Service; Islamic State in Iraq and Syria',
391 'IW': 'Information Warfare',
392 'IQT': 'In-Q-Tel (Private firm providing information technology to the CIA)',
393 'James 8. Corney': 'Deliberate misspelling of "James B. Comey"',
394 'JA': 'Julian Assange (Founder of Wikileaks)',
395 'JB': 'John Brennan (CIA Director); Joe Biden (VP under Obama); Jim Baker (FBI General Counsel); Jeff Bezos (CEO of Amazon)',
396 'JC': 'James Comey (FBI Director); James Clapper (DNI under Obama); John Carlin (Assistant AG); Josh Campbell (FBI Special Agent)',
397 'JD': 'Jack Dorsey (CEO of Twitter)',
398 'JK': 'John Kerry (Secretary of State under Obama), Jared Kushner (Senior Adviser under Trump)',
399 'JL': 'John Legend (American singer/songwriter)',
400 'John M': 'John McCain (Sen. R-AZ)',
401 'JP': 'John Podesta (WH Chief of Staff under Clinton, Counselor under Obama)',
402 'JR': 'Junior; Jim Rybicki (FBI Chief of Staff under Comey, fired by Wray)',
403 'JS': 'Jeff Sessions (US Attorney General under Trump); John Solomon (investigative reporter)',
404 'Judge K': 'Brett Kavanaugh (Supreme Court Associate Justice)',
405 'Justice K': 'Brett Kavanaugh (Supreme Court Associate Justice)',
406 'KAV': 'Brett Kavanaugh (Supreme Court Associate Justice)',
407 'KC': 'Kevin Clinesmith (FBI attorney)',
408 'KKK': 'Klu Klux Klan (created by the Democrats)',
409 'KM': 'Kelly Magsamen (Special Assistant to the President)',
410 'LARP': 'Live Action Role Player',
411 'LifeLog': 'Pentagon DARPA mass-surveillance project rebranded as Facebook',
412 'LdR': '(Lady) Lynn Forester de Rothschild (married to Evelyn de Rothschild)',
413 'LDR': '(Lady) Lynn Forester de Rothschild',
414 'LL': 'Loretta Lynch (US Attorney General under Obama)',
415 'LLC': 'Limited Liability Company',
416 'LP': 'Lisa Page (FBI Special Counsel)',
417 'LYNCH': 'Loretta Lynch (US Attorney General under Obama)',
418 'M’s': 'Marines',
419 'Maxine W': 'Maxine Waters (Rep. D-CA)',
420 'Mc_I': 'John McCain Institute',
421 'MACRON': 'Emmanuel Macron (President of France)',
422 'MAGA': 'Make America Great Again',
423 'MAY': 'Theresa May (Prime Minster of UK)',
424 'MB': 'Muslim Brotherhood',
425 'MCCABE': 'Andrew McCabe (FBI Deputy Director under Comey, fired)',
426 'MERKEL': 'Angela Merkel (Chancellor of Germany)',
427 'MI': 'Military Intelligence',
428 'MIL': 'Military',
429 'MK': 'Mike Kortan (FBI Assistant Director)',
430 'ML': 'Martial Law',
431 'MLK': 'Martin Luther King (Civil rights advocate murdered in 1968)',
432 'MM': 'Mary McCord (Principal Deputy Assistant AG); Media Matters',
433 'MO': 'Michelle Obama (transvestite husband of Barack Obama)',
434 'MOAB': 'Mother Of All Bombs',
435 'MS': 'Microsoft; Michael Steinbach (FBI Executive Assistant Director)',
436 'MS13': 'Latino Drug Cartel; MSM',
437 'MUELLER': 'Robert Mueller (6th FBI Director)',
438 'MURKOWSKI': 'Lisa Murkowski (Sen. R-AK)',
439 'MW': 'Maxine Waters (Rep. D-CA)',
440 'MZ': 'Mark Zuckerberg (CEO of Facebook)',
441 'N&S': 'North and South',
442 'N1LB': 'No One Left Behind',
443 'No Name': 'John McCain',
444 'No Such Agency': 'NSA',
445 'NAT': 'National',
446 'NATSEC': 'National Security',
447 'NOFORN': 'No Foreign Nationals (Document Sensitivity Level)',
448 'NO NAME': 'John McCain',
449 'NO SUCH AGENCY': 'NSA',
450 'NP': 'Nancy Pelosi (Rep. D-CA); Non-Profit',
451 'NPO': 'Non-Profit Organization',
452 'NR': 'Nuclear Reactor; Nuclear Radiation',
453 'NSC': 'National Security Council',
454 'NUNES': 'Devin Nunes (Rep. R-CA)',
455 'NWO': 'New World Order; Nazi World Order (?)',
456 'NXIVM': 'Sex trafficking cult with close ties to Democratic Party',
457 'OO': 'Oval Office (White House)',
458 'OP': 'Operation; Operator; Original Post; Original Poster; Operated Plane',
459 'OPs': 'Operations',
460 'OS': 'Oversight',
461 'OWL': 'Orbital Weapon Lancet (Space-based weapon) (?)',
462 'P': 'POTUS; Presidential; Pope; Peninsula; Paragraph; Page; Payseur',
463 'P_Pers': 'POTUS Personal',
464 'PAC': 'Political Action Committee',
465 'PAGE': 'Lisa Page (FBI Special Counsel)',
466 'PANIC': 'Patriots Are Now In Control',
467 'PD': 'Police Department',
468 'PDB': 'President’s Daily Brief',
469 'PDBs': 'President’s Daily Briefs',
470 'PELOSI': 'Nancy Pelosi (Rep. D-CA)',
471 'PENCE': 'Mike Pence (VP under Trump)',
472 'PEOC': 'Presidential Emergency Operations Center',
473 'PG': 'Pizzagate/Pedogate',
474 'PL': 'Presidential Library',
475 'PM': 'Prime Minister; Post Meridiem; Paul Manafort (Campaign manager for Trump)',
476 'PODESTA': 'John Podesta (WH Chief of Staff under Clinton, Counselor under Obama)',
477 'POS': 'Piece Of Shit',
478 'POV': 'Point Of View',
479 'POVs': 'Points Of View',
480 'PP': 'Planned Parenthood',
481 'PRISM': 'NSA Internet data collection program',
482 'PS': 'Peter Strzok (FBI Lead Agent); PlayStation',
483 'PST': 'Pacific Standard Time',
484 'PTSD': 'Post-Traumatic Stress Disorder',
485 'PUTIN': 'Vladimir Putin (President of Russia)',
486 'Q&A': 'Question and Answer',
487 'Q+': 'President Trump; Q-team',
488 'R v W': 'Right versus Wrong',
489 'R\'s': 'Republicans',
490 'R’s': 'Republicans',
491 'R+D': 'Republican + Democrat',
492 'RB': 'Rachel Brand (Associate AG)',
493 'RBG': 'Ruth Bader Ginsburg (Supreme Court Associate Justice)',
494 'RC': 'Rachel Chandler (Child handler for Jeffrey Epstein who did not kill himself)',
495 'RE': 'Rahm Emanuel (White House Chief of Staff under Obama)',
496 'RED RED': 'Red Cross',
497 'RED_RED': 'Red Cross',
498 'RENEGADE': 'Secret Service codename for Barack Hussein Obama',
499 'RICE': 'Susan Rice (National Security Advisor under Obama)',
500 'RIP': 'Rest In Peace',
501 'RM': 'Robert Mueller (6th FBI Director)',
502 'RNC': 'Republican National Committee',
503 'RR': 'Rod Rosenstein (Deputy Attorney General under Trump)',
504 'RT': 'Real Time; Retweet; Rex Tillerson (Secretary of State under Trump)',
505 'RUDY': 'Rudy Giuliani (former Mayor of NYC)',
506 'SB': 'Senate Bill',
507 'SAP': 'Special Access Program',
508 'SAPs': 'Special Access Programs',
509 'Scaramucci': 'Anthony Scaramucci (WH Communications Director under Trump, fired after repeated TDS-attacks on Trump)',
510 'SCHUMER': 'Chuck Schumer (Sen. D-NY)',
511 'SCI': 'Sensitive Compartmented Information (TOP SECRET+)',
512 'SCIF': 'Sensitive Compartmented Information Facility',
513 'SDNY': 'Southern District of New York',
514 'SEC': 'Security; Section',
515 'SEC_TEST': 'Security Test',
516 'SESSIONS': 'Jeff Sessions (US Attorney General under Trump)',
517 'SH': 'Sean Hannity (Conservative TV host); Steve Huffman (CEO of Reddit)',
518 'SIGINT': 'Signals Intelligence',
519 'SIT ROOM': 'Situation Room (White House)',
520 'SM': 'Sally Moyer',
521 'SMOLLETT': 'Jussie Smollett (Hollywood actor who faked his own lynching)',
522 'SOROS': 'George Soros (Billionaire globalist investor)',
523 'SOTU': 'State Of The Union',
524 'SP': 'Samantha Power (US Ambassador to the UN)',
525 'SR': 'Seth Rich (DNC staffer murdered after leaking to Wikileaks); Susan Rice (National Security Advisor under Obama); Senior',
526 'ST': 'Shit (?)',
527 'STEELE': 'Christopher Steele (MI6 agent who concocted the Steele-dossier)',
528 'STRAT': 'Strategic',
529 'STRZOK': 'Peter Strzok (FBI agent who participated in the attempted subversion of the 2016 presidential election)',
530 'SURV': 'Surveillance',
531 'SY': 'Sally Yates (Deputy Attorney General)',
532 'TBA': 'To Be Announced',
533 'TG': 'Trey Gowdy (Rep. D-SC); Tashina Gauhar (FISA lawyer)',
534 'TM': 'Team',
535 'TP': 'Tony Podesta (Brother of John Podesta)',
536 'TRI': 'Trilateral Commission (?)',
537 'TT': 'Trump Tower; Tarmac Tapes',
538 'T-Tower': 'Trump Tower',
539 'U1': 'Uranium One',
540 'UBL': 'Usama Bin Laden',
541 'UC': 'Univerity of California',
542 'USD': 'US Dollar',
543 'USMIL': 'US Military',
544 'VIP': 'Very Important Person',
545 'VIPs': 'Very Important Persons',
546 'VJ': 'Valerie Jarret (Senior Advisor to Obama)',
547 'W&W': 'Wizards and Warlocks',
548 'WHITAKER': 'Matthew G. Whitaker (Acting US Attorney General after Sessions resigned)',
549 'WIA': 'Wounded In Action',
550 'WMDs': 'Weapons of Mass Destruction',
551 'WRAY': 'Christopher Wray (8th FBI Director)',
552 'WRWY': 'We Are With You',
553 'WW': 'World Wide; World War',
554 'WWI': 'World War 1',
555 'WWII': 'World War 2',
556 'WWIII': 'World War 3',
557 'WWG1WGA': 'Where We Go One We Go All',
558 'XKeyscore': 'NSA Internet data search and analysis tool'
559}
560
561#################
562# Methods for Qposts Research:
563
564def qmap_id_to_qpost_index( qmap_id ):
565 '''<qmap_id> can either be a Qmap ID, or a tuple/list of Qmap IDs.
566 Qmap IDs are 1-based and run from oldest to latest, while Qpost index numbers are 0-based and run from latest to oldest.'''
567 if isinstance( qmap_id, ( tuple, list ) ): return [ len( _QPOSTS ) - idx for idx in qmap_id ]
568 return len( _QPOSTS ) - qmap_id
569
570
571def qpost_index_to_qmap_id( qpost_idx ):
572 '''For clarity; Same output as qmap_id_to_qpost_index( qpost_idx ).'''
573 return qmap_id_to_qpost_index( qpost_idx )
574
575
576def is_valid_qmap_id( qmap_id ):
577 '''Returns True if <qmap_id> is a valid Qmap ID number.'''
578 return qmap_id > 0 and qmap_id <= len( _QPOSTS )
579
580
581def open_qposts_json( url_qposts_json ):
582 '''Loads the specified JSON-file, and returns a list of dictionaries, or None.'''
583 safe_url = collapse_user( url_qposts_json ) # Remove user name before printing.
584 if os.path.exists( url_qposts_json ):
585 try:
586 with open( url_qposts_json, 'r', encoding='utf-8' ) as qps_file:
587 return json.loads( qps_file.read() )
588 except: print( f"JSON: Failed to load Qdata from file '{safe_url}'." )
589 else: print( f"Error: No such file: '{safe_url}'." )
590
591
592def download_qposts_json( url_save ):
593 '''Downloads the Qposts JSON-file from qanon.pub, and saves it to the specified url.
594 This function returns a list of dictionaries taken from the downloaded JSON-file, or None.'''
595 _URL_JSON = "https://qanon.pub/data/json/posts.json"
596 try: # DOWNLOAD JSON DATA FROM QANON.PUB.
597 req = Request( _URL_JSON, headers={'User-Agent': 'Mozilla/5.0'} )
598 qdata = urlopen( req )
599 except Exception as e: print( f'{er}Failed to download json file "{_URL_JSON}":{ts} {e}' ); return None
600 try: qposts = json.loads( qdata.read().decode() )
601 except Exception as e: print( f'{er}JSON: Failed to load Qpost data from downloaded file;{ts} {e}' ); return None
602 safe_url = collapse_user( url_save ) # Hide user name before printing.
603 try: # DUMP DATA TO JSON FILE.
604 with open( url_save, "w", encoding="utf-8" ) as qps_file:
605 json.dump( qposts, qps_file, ensure_ascii=False, indent=2 )
606 except Exception as e: print( f'{er}JSON: Failed to save Qpost data to file "{safe_url}":{ts} {e}' ); return None
607 return qposts
608
609
610def download_latest_qpost_nr():
611 '''Downloads a (limited) XML RSS-file from qalerts.app.
612 Returns a 2-tuple with the number of the most recent Qpost from qalerts.app, plus the root node to an XML ElementTree
613 containing (limited) records of the 20 most recent Qposts from qalerts.app; Returns (None,None) if something went wrong.
614 NB. The latest Qpost on qalerts.app is not necessarily available at exactly the same moment as the qposts.json at qanon.pub.'''
615 # import xml.etree.ElementTree as xml_tree
616 try: # DOWNLOAD RSS-DATA FROM QALERTS.APP
617 _URL_RSS = "https://qalerts.app/data/rss/posts.rss"
618 req = Request( _URL_RSS, headers={ 'User-Agent': 'Mozilla/5.0' } )
619 qdata = urlopen( req )
620 except Exception: return None, None
621 root = xml_tree.fromstring( qdata.read().decode() )
622 try: # PARSE RSS-DATA
623 text = root[0][10][0].text
624 return int( text.replace( 'Q Drop #', '' ) ), root
625 except: return None, root
626
627
628def check_update_local_qposts():
629 '''This function checks online if there are new Qposts available, and if so, it offers to download them.
630 The local Qposts.json file will be overwritten; if there exists a local Qposts.db database file,
631 it will be updated by adding only new records for the new Qposts.
632 Returns the updated current number of Qposts inside the local Qposts.json file.'''
633 global _QPOSTS
634 n_qposts = len( _QPOSTS )
635 l_qpost, rss_root = download_latest_qpost_nr() # Check for latest Qposts online.
636 if l_qpost:
637 if l_qpost > n_qposts: # NEW Qpost(s) available!
638 n_new = l_qpost - n_qposts
639 answer = input( f"{tc}[5m?{ts+tc} There {'is' if n_new == 1 else 'are'} {bw} {n_new} {ts+tc} New Qpost" +
640 f"{'' if n_new == 1 else 's'} available!{ts}\n{mc}Do you want to update your local Qposts now? (Y/n):{ts} " )
641 if answer.lower() not in ['n', 'no']: # Update Qposts.json, and Qposts.db if present.
642 print( f"{tc}Updating local Qposts:{ts} Attempting to download Qposts.json from qanon.pub ..." )
643 _QPOSTS = download_qposts_json( _URL_QPOSTS )
644 n_qposts = len( _QPOSTS )
645 if os.path.exists( _URL_QPOSTS_DB ): # Update Qposts.db if present.
646 qposts_to_sqlite( _QPOSTS, _URL_QPOSTS_DB )
647 n_recs = qposts_sqlite_count_records( _URL_QPOSTS_DB )
648 elif l_qpost == n_qposts:
649 print( f'{ok}Online check completed: Your local Qposts.json is already up to date.{ts}' )
650 else: print( f'{er}Could not check online if there are any new Qposts available.{ts}' )
651 return n_qposts
652
653
654def get_qpost_media_urls( qpost ):
655 '''Returns a nested list of tuples(url,filename) for all media linked in <qpost> and in its references recursively.'''
656 tuples = []
657 if isinstance( qpost, dict ):
658 qpost_media = qpost.get( 'media', [] )
659 if qpost_media:
660 for image in qpost_media:
661 tuples.append( ( image.get( 'url', '' ), image.get( 'filename', '' ) ) )
662 qpost_refs = qpost.get( 'references', [] ) # this tag is only present if the Qpost has references.
663 if qpost_refs:
664 for qp_ref in qpost_refs:
665 tuples.append( get_qpost_media_urls( qp_ref ) )
666 return tuples
667
668
669def download_qpost_images( qmap_ids_list, references_too=False ):
670 '''Workaround; Tries to download the media linked in all Qposts whose Qmap ID is specified in <qmap_ids_list>;
671 This creates a subfolder for each Qmap ID that has linked media, and saves all downloaded media for that Qmap ID inside that subfolder.
672 Subfolders for downloaded media will all be created inside the folder <_URL_QPOSTS_IMAGES>, and will be named after the Qmap ID number.
673 Qposts can contain references, that can in turn also contain media. To recursively download these media too, pass <references_too>=True.
674 NB. this could result in the downloading of multiple duplicate image files, when a Qpost with images is referenced in another Qpost.
675 Statistics for 3774 Qposts:
676 excl. References: total 809 media; 694 subfolders; DL size ~370MB; DL time ~~20min; DL success-rate 100%.
677 incl. References: total 1941 media; 1273 subfolders; DL size ~864MB; DL time ~~50min; DL success-rate ~93% (Fails for 136 files).'''
678 # import os
679 def download_urls( url_list, save_folder, references_too ):
680 '''Download files from nested list <url_list> and save them into the folder <save_folder>.'''
681 urls_to_replace = [ 'https://media.8ch.net/file_store/thumb/',
682 'https://media.8ch.net/file_store/',
683 '//media.jthnx5wyvjvzsxtu.onion/file_store/' ]
684 url_replacements = [ 'https://qalerts.app/media/',
685 'https://qposts.online/assets/images/' ]
686 total_media, dl_success, already_present = 0, 0, 0
687 for item in url_list:
688 nm, ns, ap = 0, 0, 0
689 if isinstance( item, tuple ) and len(item) == 2:
690 total_media += 1
691 image_url, filename = item
692 image_url_rep = image_url
693 filename2 = os.path.split( image_url )[1] # storage name.
694 if not filename: filename = filename2
695 save_url = os.path.join( save_folder, filename )
696 if not os.path.exists( save_url ):
697
698 # Added since 8chan is offline, making all 8ch.net image-urls invalid;
699 # instead try to download the corresponding images from qposts.online or qalerts.app.
700 # TODO: change this part when 8kun has transfered all the older images from 8ch.net.
701 for url_replacement in url_replacements:
702 for url in urls_to_replace:
703 if image_url.startswith( url ):
704 image_url_rep = image_url.replace( url, url_replacement ); break
705 save_ok = download_binary_file( image_url_rep, save_url ) # Download/Save media.
706 if not save_ok: # Download failed:
707 fparent, fname = os.path.split( image_url_rep ) # Retry with filename2.
708 image_url_rep = os.path.join( fparent, filename2 )
709 save_ok = download_binary_file( image_url_rep, save_url )
710 if save_ok: break
711
712 dl_success += bool( save_ok )
713 url_safe = collapse_user( save_url )
714 if save_ok: print( f"{ok}Success:{ts} media '{image_url_rep}'\n\t{ok} was saved{ts} to file '{url_safe}'." )
715 else: print( f"{er}Failed:{ts} media '{image_url_rep}'\n\t{er} was not saved{ts} to file '{url_safe}'." )
716 else: already_present += 1
717 elif isinstance( item, list ) and references_too:
718 nm, ns, ap = download_urls( item, save_folder, references_too )
719 total_media += nm; dl_success += ns; already_present += ap
720 return total_media, dl_success, already_present
721 ###### END of download_urls().
722
723 if not os.path.exists( _URL_QPOSTS_IMAGES ): os.mkdir( _URL_QPOSTS_IMAGES )
724 tuples = get_qposts_by_id( qmap_ids_list, key='', qmap_ids=True )
725 total_qposts, total_media, qp_has_media, success, already_present = 0, 0, 0, 0, 0
726 for qmap_id, qpost in tuples:
727 total_qposts += 1
728 qpost_media = get_qpost_media_urls( qpost ) # Nested list of tuples(url,filename).
729 qpost_own_media, qpost_ref_media = split_list( qpost_media )
730 if qpost_own_media or qpost_ref_media:
731 qp_has_media += 1
732 folder_qmap_id = os.path.join( _URL_QPOSTS_IMAGES, f'{qmap_id}', '')
733 if not os.path.exists( folder_qmap_id ):
734 if references_too or qpost_own_media: os.mkdir( folder_qmap_id ) # Create a subfolder for this QmapID.
735 nm, ns, ap = download_urls( qpost_media, folder_qmap_id, references_too ) # Download Qpost media into subfolder.
736 total_media += nm; success += ns; already_present += ap
737 n_failed = total_media - already_present - success
738 print( f"{tc}Processed {bw} {total_qposts} {tc} Qposts ({['excl.','incl.'][references_too]} references);{ts}\n{bw} {qp_has_media} {tc} " +
739 f"Qposts contain a total of {bw} {total_media} {tc} linked media:{ts}\n{tc}Media Present:{ts} {already_present}/{total_media}" +
740 f"{tc} Media Downloaded: {ok} {success}/{total_media} {tc} Download Failed: {er} {n_failed}/{total_media} {tc}.{ts}" )
741
742
743def get_qposts_by_id( qpost_ids, key='', qmap_ids=False ):
744 '''Returns a list with tuples(qmap_id,element) of all qposts whose indexes are specified in <qpost_ids>.
745 <qpost_ids>: List of indexes into the _QPOSTS list ( or list of Qmap IDs if <qmap_ids>=True ).
746 <key>: if given, the returned list elements will be only the field qpost[key]; else they will be the entire qpost dict.
747 <qmap_ids>: if True, the given <qpost_ids> are interpreted as Qmap IDs ranging from 1 to N ( where 1 is the first Qpost),
748 else they are interpreted as indexes into the _QPOSTS list, ranging from 0 to N-1 ( where 0 is the latest Qpost).'''
749 if _QPOSTS:
750 qp_count = len( _QPOSTS )
751 result = []
752 for i in qpost_ids:
753 qmap_id = i if qmap_ids else qp_count - i
754 index = qp_count - i if qmap_ids else i
755 qpost = _QPOSTS[index]
756 if key: result.append( ( qmap_id, qpost.get(key) ) )
757 else: result.append( ( qmap_id, qpost ) )
758 return result
759
760
761def search_qposts_regexp( regexp, key='' ):
762 '''Returns a list with tuples(qmap_id,qpost) of all qpost dicts whose <key>-field matches <regexp>.
763 <regexp>: String representing the regular expression to match.
764 <key>: Pass a qpost dict key, or pass "" to search the entire qpost dict as string.
765 Valid keys: see _QPOST_KEYS.'''
766 # import re
767 result = []
768 for i, qpost in enumerate( _QPOSTS ):
769 find_in = qpost[key] if qpost.get( key ) else str(qpost)
770 if re.search( regexp, find_in ): result.append( ( qpost_index_to_qmap_id(i), qpost ) )
771 return result
772
773
774def search_qposts( find, key='text' ):
775 '''Returns a list with tuples(qmap_id,qpost) of all qposts whose <key>-field contains <find>.
776 <find>: can be a complex search term using operators (and, or, not) and parentheses; complex atoms must be double-quoted.
777 If <key> is empty, it searches the text of the whole qpost dictionary for <find>; Some keys may not be present in all qposts.
778 Valid keys: see _QPOST_KEYS.'''
779 result = []
780 x = Simple_Logic_Expression( find ) # class included below.
781 for i, qpost in enumerate(_QPOSTS):
782 text = qpost.get( key, '' ) if key else qpost
783 if not isinstance( text, str ): text = str( text )
784 if text and x.find_in( text ):
785 result.append( ( qpost_index_to_qmap_id(i), qpost ) )
786 return result
787
788
789def search_qposts_by_date( date_condition ):
790 '''Returns a list with tuples(Qmap_id, Qpost) of all qposts whose timestamp matches <date_condition>.
791 <date_condition>: String of the format "COMP DATETIME", where COMP is one of the comparison operators in:
792 [ 'on', '>=', '<=', '!=', '==', '=', '>', '<' ], and DATETIME is a valid datetime expression such as '1 Aug 2019',
793 or a timestamp number. Each "COMP DATETIME" pair must be enclosed within double quotation marks,
794 and multiple "COMP DATETIME" pairs can be combined using logical connectives [and,or] and parentheses.
795 for example: ("> 1 Aug 2019" and "< 2 Aug 2019") or "> 1 Jan 2020" .'''
796 result = []
797 x = Simple_Logic_Expression( date_condition ) # class included below.
798 for i, qpost in enumerate(_QPOSTS):
799 timestamp = qpost.get( 'timestamp', -1 )
800 if not isinstance( timestamp, (int,float) ): timestamp = float( timestamp )
801 if timestamp and x.match_value( timestamp, evaluate_datetime_condition ): # pass function as arg.
802 result.append( ( qpost_index_to_qmap_id(i), qpost ) )
803 return result
804
805
806def get_qposts_for_date( d ):
807 '''Returns a list of tuples(QmapID, Qpost) whose Qpost timestamp falls on the specified day <d>.
808 <d>: datetime.datetime object representing the date for which to return all Qposts that were posted on the same day.'''
809 return search_qposts_by_date( d.timestamp() )
810
811
812def get_qposts_for_dates( dates ):
813 '''Returns a list of tuples(QmapID, Qpost) whose Qpost timestamp falls on any of the dates specified in <dates>.
814 <dates>: list of datetime.datetime objects representing the dates for which to return all Qposts.'''
815 result = []
816 for d in dates:
817 result.extend( get_qposts_for_date( d ) )
818 return result
819
820
821def get_qpost_dates_for_qmap_ids( qmap_ids_list ):
822 '''Returns a list of datetime objects representing the posting dates of the Qposts specified in <qmap_ids_list>.'''
823 qposts = get_qposts_by_id( qmap_ids_list, key='timestamp', qmap_ids=True )
824 return [ datetime.datetime.fromtimestamp( float( qp[1] ), tz=None ) for qp in qposts ]
825
826
827def qclock_get_aligned_dates_for_date( qdate=datetime.datetime.today(), include_mirror=True ):
828 '''Collect earlier dates from the Qclock, that are located on the same radius or diameter line as the date specified in <qdate>.
829 <qdate>: a datetime.datetime object or a string representing the date for which to retrieve the earlier Qclock-aligned dates;
830 When passing a date string, prevent ambiguities by putting the day number before the month, and passing a 4-digit year.
831 <include_mirror>: Boolean determining whether to also include the aligned dates from the opposite side of the Qclock center.
832 Returns a tuple with 4 elements: (the parsed input date; a standard string representation of the parsed input date; a list containing
833 the aligned dates on this side of the center; and a list containing the aligned dates on the opposite side of the center: this latter
834 list will be empty if <include_mirror>=False).'''
835 # import datetime; _DT_FORMAT_L = '%A %d %B %Y'
836 if isinstance( qdate, str ): dt, dts, _ = parse_datetime_string( qdate, _DT_FORMAT_L )
837 elif isinstance( qdate, datetime.datetime ): dt, dts = qdate, qdate.strftime( _DT_FORMAT_L )
838 if not dt: return None, '', [], []
839 else: d_target = dt.toordinal() # Serial Day Number of input date.
840 aligned, aligned_mirror, current, mirror = [], [], d_target, d_target - 30
841 d_start = 736630 # QClock Start: '10-28-17'; ( hour, angle, minutes ) = ( 4, 10, 20 ) = DJT.
842 while current >= d_start:
843 aligned.append( datetime.datetime.fromordinal( current ) ); current -= 60
844 if include_mirror:
845 while mirror >= d_start:
846 aligned_mirror.append( datetime.datetime.fromordinal( mirror ) ); mirror -= 60
847 return dt, dts, aligned, aligned_mirror
848
849
850def qclock_get_aligned_dates_for_clocktime( qtime=datetime.datetime.now(), round_to_nearest=True ):
851 '''Collect all dates between 10-28-2017 and today, that are located on either of the Qclock hands at the specified clock time.
852 <qtime>: A datetime.datetime, datetime.time, or a 2-tuple(H,M) specifying the digital time for which to retrieve the aligned dates.
853 NB. For a digital time of 04:20 the hour-hand on the analog clock is pointing at precisely 21.67 minutes, which would be
854 returned here as the 22nd minute if <round_to_nearest>=True, else as the 21st minute;
855 Returns a tuple with 2 lists containing datetime.datetime objects: the first list contains the dates located on the hour-hand, and the
856 second list contains the dates located on the minute-hand of the Qclock.'''
857 # import datetime
858 hour_hand, minute_hand, roundoff = [], [], [int,round][bool(round_to_nearest)]
859 d_today = datetime.datetime.today().toordinal()
860 d_start = 736630 # QClock start date: '10-28-2017'
861 if isinstance( qtime, ( datetime.datetime, datetime.time ) ): H, M = qtime.hour, qtime.minute
862 elif isinstance( qtime, (tuple,list) ) and len( qtime ) > 1: H, M = qtime[0], qtime[1]
863 else: return [], []
864 try: H, M = int( H ) % 12, int( M ) % 60
865 except: return [], []
866 H_date = d_start + roundoff( ( ( H + 8 ) % 12 + M / 60 ) * 5 ) # rounded to floor or nearest.
867 M_date = d_start + ( M + 40 ) % 60
868 while H_date <= d_today:
869 hour_hand.append( datetime.datetime.fromordinal( H_date ) ); H_date += 60
870 while M_date <= d_today:
871 minute_hand.append( datetime.datetime.fromordinal( M_date ) ); M_date += 60
872 return hour_hand, minute_hand
873
874
875def qclock_get_aligned_qposts_for_date( qdate=datetime.datetime.today(), include_mirror=True, print_list=True ):
876 '''Collects all earlier Qposts posted on a date that aligns with the given <qdate> on the Qclock.
877 <qdate>: a datetime.datetime object or a string representing the date for which to retrieve all earlier aligned Qposts.
878 <include_mirror>: Boolean determining whether to also include the aligned Qposts from the opposite side of the Qclock center.
879 <print_list>: If True, it prints out the aligned dates and the number of Qposts that were posted on each of those dates.
880 Returns a single list of tuples(QmapID, Qpost) for all Qposts whose date aligns on the Qclock with the input date.'''
881 # import datetime
882 dt, dts, aligned, mirror = qclock_get_aligned_dates_for_date( qdate, include_mirror )
883 if dts:
884 result, s_aligned, s_mirror = [], [], []
885 for d in aligned:
886 qposts_for_day = get_qposts_for_date( d ); n_qp = len(qposts_for_day)
887 result.extend( qposts_for_day )
888 if print_list: s_aligned.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
889 if print_list: print( f'{tc}Qclock Directly aligned dates for {bw} {dts} {ts+tc}:{ts}\n{", ".join( s_aligned )}' )
890 if include_mirror:
891 for d in mirror:
892 qposts_for_day = get_qposts_for_date( d ); n_qp = len(qposts_for_day)
893 result.extend( qposts_for_day )
894 if print_list: s_mirror.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
895 if print_list: print( f'{tc}Qclock Opposite aligned dates for {bw} {dts} {ts+tc}:{ts}\n{", ".join( s_mirror )}' )
896 return result
897
898
899def qclock_get_aligned_qposts_for_clocktime( qtime, round_to_nearest=True, print_list=True ):
900 '''Returns a list of tuples(QmapID, Qpost) for all Qposts whose date is located on one of the hands of the Qclock on the given clocktime.
901 <qtime>: A datetime.datetime, datetime.time, or a 2-tuple(H,M) specifying the digital time for which to retrieve the Qclock-aligned Qposts.
902 <print_list>: If True, it prints out the dates on the Qclock-hands, and the number of Qposts that were posted on each of those dates.'''
903 result, s_hour, s_minute = [], [], []
904 hour_hand, minute_hand = qclock_get_aligned_dates_for_clocktime( qtime, round_to_nearest )
905 for d in hour_hand:
906 qposts_for_day = get_qposts_for_date( d ); n_qp = len( qposts_for_day )
907 result.extend( qposts_for_day )
908 if print_list: s_hour.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
909 for d in minute_hand:
910 qposts_for_day = get_qposts_for_date( d ); n_qp = len( qposts_for_day )
911 if hour_hand != minute_hand: result.extend( qposts_for_day )
912 if print_list: s_minute.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
913 if print_list:
914 print( f'{tc}Qclock Hour-hand dates for clocktime ({qtime[0]:02d}:{qtime[1]:02d}):{ts}\n{", ".join( s_hour )}' )
915 print( f'{tc}Qclock Minute-hand dates for clocktime ({qtime[0]:02d}:{qtime[1]:02d}):{ts}\n{", ".join( s_minute )}' )
916 return result
917
918
919def qclock_get_aligned_qposts_for_qmap_id( qmap_id, include_mirror=True, print_list=True ):
920 '''Collects all earlier Qposts aligning on the Qclock with the Qpost of the given <qmap_id>.
921 <qmap_id>: Integer Qmap ID number of the Qpost for which to get all Qclock-aligned Qposts.
922 <include_mirror>: Boolean determining whether to also include the aligned dates from the opposite side of the Qclock center.
923 <print_list>: If True, it prints out the aligned dates and the number of Qposts that were posted on each of those dates.
924 Returns a list of tuples(QmapID, Qpost) for all Qposts whose date aligns on the Qclock with the post date of the given <qmap_id>.'''
925 return qclock_get_aligned_qposts_for_date( get_qpost_dates_for_qmap_ids( [ qmap_id ] )[0], include_mirror, print_list )
926
927
928def qposts_to_sqlite( qposts, url_save ):
929 '''Exports a list of Qpost-dictionaries into an SQLite3 database; Only non-existing (new) records are added to the database.'''
930 # import os, sqlite3
931 try:
932 qposts_reversed = qposts.copy()
933 qposts_reversed.reverse() # Reverse order of Qposts, so that the oldest Qpost gets qmap_id=1.
934 connection = sqlite3.connect( url_save ) # Creates an empty db if the specified name is not found.
935 cursor = connection.cursor()
936 success = 0
937 str_sql = "CREATE TABLE IF NOT EXISTS qposts (qmap_id INTEGER PRIMARY KEY, timestamp INTEGER, text TEXT, media TEXT, "
938 str_sql += "refs TEXT, name TEXT, trip TEXT, userId TEXT, link TEXT, source TEXT, threadId TEXT, id TEXT, title TEXT, "
939 str_sql += "subject TEXT, email TEXT, timestampDeletion INTEGER, CONSTRAINT unique_id_ts UNIQUE (id,timestamp));"
940 cursor.execute( str_sql ) # Create table "qposts" if not already present.
941 connection.commit()
942 for qp in qposts_reversed: # Add only the *new* qposts to the table.
943 values = ( None, int( qp.get( 'timestamp', -1 ) ), qp.get( 'text', '' ), str( qp.get( 'media', '' ) ),
944 str( qp.get( 'references', '' ) ), qp.get( 'name', '' ), qp.get( 'trip', '' ), qp.get( 'userId', '' ),
945 qp.get( 'link', '' ), qp.get( 'source', '' ), qp.get( 'threadId', '' ), qp.get( 'id', '' ),
946 qp.get( 'title', '' ), qp.get( 'subject', '' ), qp.get( 'email', '' ), qp.get( 'timestampDeletion', None ) )
947 try: cursor.execute( "INSERT INTO qposts VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );", values ); success += 1
948 except: pass
949 connection.commit()
950 connection.close()
951 filename = os.path.split( url_save )[1]
952 hlink = ansi_hyperlink( f'file://{url_save}', filename, fg=(150,190,10), bg=(), style=(0,1,1,0,0) )
953 print( f'{tc}Exported {bw} {success} {ts+tc} new Qpost records to SQLite3 Database "{hlink}{tc}".{ts}' )
954 except Exception as e:
955 safe_url = collapse_user( url_save ) # Hide user name before printing url.
956 print( f'{er}Error while exporting Qpost records to SQLite3 Database "{safe_url}":{ts} {e}' )
957
958
959def qposts_sqlite_query( url_db, sql='SELECT * FROM qposts;' ):
960 '''Executes a SELECT query <sql> in the SQLite3 database <url_db>, and returns the result.'''
961 # import os, sqlite3
962 result = '<Error>'
963 if os.path.exists( url_db ):
964 try:
965 connection = sqlite3.connect( url_db )
966 cursor = connection.cursor()
967 result = list( cursor.execute( sql ) )
968 connection.close()
969 except:
970 safe_url = collapse_user( url_db ) # Hide user name before printing url.
971 print( f'{er}SQL: Failed to execute query "{sql}" in SQLite3 Database "{safe_url}".{ts}' )
972 return result
973
974
975def qposts_sqlite_count_records( url_db, table='qposts' ):
976 '''Returns the number of records in the table <table> of the SQLite3 database <url_db>.'''
977 rec_count = 0
978 try:
979 connection = sqlite3.connect( url_db )
980 cursor = connection.cursor()
981 cursor_ids = cursor.execute( f"SELECT * FROM {table};" )
982 rec_count = len( list( cursor_ids ) )
983 connection.close()
984 except: print( f'{er}Failed to count records in SQLite3 Database table "{table}".{ts}' )
985 return rec_count
986
987
988def qpost_text_cleanup( qp_text ):
989 '''Cleanup leftover Html-formatting from the Qpost text field <qp_text>.'''
990 # import re, html
991 if qp_text:
992 if qp_text[-1] == '\n': qp_text = qp_text[:-1] # remove single newline character at the end.
993 qp_text = re.sub( r'<strong>(.*?)</strong>', r'[1m\1[0m', qp_text ) # strong-tags --> bold.
994 qp_text = re.sub( r'<em>(.*?)</em>', r'[3m\1[23m', qp_text ) # em-tags --> italic.
995 qp_text = re.sub( r'<u>(.*?)</u>', r'[4m\1[24m', qp_text ) # u-tags --> underline.
996 qp_text = re.sub( r'<span class="heading">(.*?)</span>', r'[1;38;2;175;10;15m\1[0m', qp_text ) # headings --> red bold.
997 qp_text = re.sub( r'<span class="spoiler">(.*?)</span>', r'[2m\1[22m', qp_text ) # spoilers --> dimmed text.
998 qp_text = re.sub( r'<span class="detected">(.*?)</span>', r'[7m\1[27m', qp_text ) # detected --> inverted color.
999 qp_text = re.sub( r'<p class="body-line empty ">(.*?)</p>', r'\1', qp_text ) #body-line empty --> plain text.
1000
1001 qp_text = re.sub( r'</p><p class="body-line ltr quote">', '', qp_text ) # remove faulty tag.
1002 qp_text = re.sub( r'</p><p class="body-line ltr ">', '', qp_text ) # remove faulty tag.
1003 return html.unescape( qp_text ) # Remove HTML escape-codes ( requires Python 3.4 )
1004
1005
1006def print_qpost( qp, lv='' ):
1007 '''Print elements of the specified Qpost.
1008 # <qp> : dictionary representing a Qpost from Qanon.pub.
1009 # <lv> : string representing the submenu depth level: pass _LVL_INDENT per added sublevel.'''
1010
1011 # TIMESTAMP
1012 qp_timestamp = qp.get( 'timestamp', '' )
1013 qp_date = datetime.datetime.fromtimestamp( qp_timestamp )
1014 qp_datestr = qp_date.strftime( _DTM_FORMAT )
1015 qp_datediff = datetime.datetime.now() - qp_date
1016 print( f'{lv}{tc}Timestamp: {ts}{qp_timestamp}\t{tc}Date: {ts}{qp_datestr}' )
1017 print( f'{lv}{tc}Ago: {ts}{qp_datediff}' )
1018 qp_ts_delete = qp.get( 'timestampDeletion', None )
1019 if qp_ts_delete:
1020 qp_ts_datestr = datetime.datetime.fromtimestamp( qp_ts_delete ).strftime( _DTM_FORMAT )
1021 print( f'{lv}{tc}Timestamp Deletion: {ts}?{qp_ts_delete}\t{tc}Date: {ts}{qp_ts_datestr}' )
1022
1023 # NAME
1024 qp_name = qp.get( 'name', '<None>' )
1025 qp_trip = qp.get( 'trip', '<None>' )
1026 qp_userId = qp.get( 'userId', '<None>' )
1027 print( f'{lv}{tc}Name: {ts}{qp_name}\t\t\t{tc}Tripcode: {ts}{qp_trip}\t{tc}User ID: {ts}{qp_userId}' )
1028
1029 # SOURCE
1030 qp_source = qp.get( 'source', '' )
1031 qp_threadId = qp.get( 'threadId', '' ) # this tag can be missing in some posts.
1032 qp_id = qp.get( 'id', '' )
1033 qp_link = qp.get( 'link', '' )
1034 print( f'{lv}{tc}Source: {ts}{qp_source}\t{tc}Thread ID: {ts}{qp_threadId}\t{tc}Post ID: {ts}{qp_id}' )
1035 print( f'{lv}{tc}Source Link: {ts}{qp_link}' )
1036
1037 #TITLE/SUBJECT
1038 for title in [ 'subject', 'title' ]:
1039 qp_title = qp.get( title, None )
1040 if qp_title is not None: print( f'{lv}{tc}{title.title()}: {ts}{qp_title}' )
1041
1042 # TEXT
1043 qp_text = qp.get( 'text', '' )
1044 print( f'{lv}{tc}Text: {ts}' )
1045 if qp_text:
1046 qp_text = qpost_text_cleanup( qp_text )
1047
1048 if _MAX_WRAP > 0:
1049 lines = []
1050 for txline in qp_text.splitlines(): # wrap text to a fixed width.
1051 lines.extend( wrap( txline, _MAX_WRAP, break_long_words=False, break_on_hyphens=False, initial_indent=lv, subsequent_indent=lv ) )
1052 qp_text = '\n'.join( lines )
1053
1054 if _SHOW_ABBR_TOOLTIPS:
1055 for abbr in _QPOST_ABBR: # add Tooltip to known abbreviations.
1056 if abbr in qp_text:
1057 qp_text = re.sub( r"(\W|\n|^)" + re.escape(abbr) + r"(\W|\n|$)", r"\1" +
1058 ansi_hyperlink( _QPOST_ABBR[abbr], abbr ) + r"\2", qp_text )
1059
1060 print( f'{qp_text}' )
1061
1062 # MEDIA
1063 qp_media = qp.get( 'media', [] )
1064 if qp_media is None: qp_media = []
1065 print( f'{lv}{tc}Media: {ts}{len(qp_media)}' )
1066 for q_pic in qp_media:
1067 print( f"{lv}{_LVL_INDENT} {q_pic['url']}" )
1068 print( f"{lv}{_LVL_INDENT} {q_pic['filename']}" )
1069
1070 # REFERENCES
1071 qp_refs = qp.get( 'references', [] ) # this tag is only present if the Qpost has references.
1072 print( f'{lv}{tc}References: {ts}{len(qp_refs)}' )
1073 for q_ref in qp_refs:
1074 print_qpost( q_ref, lv + _LVL_INDENT )
1075
1076
1077def find_regexp_groups( tuples, regexp ):
1078 '''Returns a list of matched groups from <regexp>.
1079 <tuples>: List of tuples(QmapID,str_or_dict).
1080 <regexp>: The regular expression to match in the 2nd element of <tuples>; parenthesized groups are returned.
1081 This argument should be passed as a raw string, e.g. fr"{regexp}".'''
1082 #import re
1083 ret = []
1084 for item in tuples:
1085 s = re.findall( regexp, str(item[1]) )
1086 if s and len( s ) > 0: ret.append( (item[0], s) )
1087 return ret
1088
1089
1090def print_qpost_tuples( qp_tuples, key='' ):
1091 '''Display results returned by search_qposts(), search_qposts_by_date(), get_qposts_by_id().'''
1092 for qp in qp_tuples:
1093 print( f'\n{tc}Qmap ID:{ts} {qp[0]}' )
1094 if isinstance( qp[1], dict ): print_qpost( qp[1] )
1095 else: print( f"{tc}{key}:{ts} {qp[1] if key != 'text' else qpost_text_cleanup(qp[1]) }" )
1096
1097
1098def print_unique_qpost_field_values( qp_tuples, key='trip' ):
1099 '''Display unique values found in the Qpost field <key> of the Qposts in <qp_tuples>.'''
1100 qp_unique = set( [ qp[1] for qp in qp_tuples ] )
1101 print( f'{tc}Unique values for key=\'{key}\':{ts} {qp_unique}' )
1102
1103
1104def print_qpost_field_frequency_list( qp_tuples, key='', lex=0 ):
1105 '''Display a list of character/word frequencies found in the Qpost field <key> of the Qposts in <qp_tuples>.'''
1106 qp_freqlist = [ ( qp[0], count_frequencies( qp[1], lex=lex ) ) for qp in qp_tuples ]
1107 print( f"{tc}{('Character','Word')[min(max(0,lex),1)]} Frequency list for key=\'{key}\':{ts} {qp_freqlist}" )
1108
1109
1110def qposts_terminal_loop():
1111 '''Start an input loop in the Terminal where the user can perform various operations on the _QPOSTS list.'''
1112 global _QMAP_IDS, _QPOSTS
1113 n_posts = len(_QPOSTS)
1114 choice = 0
1115 duration_options = [ (f' {tc}1{ts}: as a number of seconds.', ['1'], ''),
1116 (f' {tc}2{ts}: in the format (H)HH:MM:SS.', ['2'], ''),
1117 (f' {tc}3{ts}: in the format DAYS, HH:MM:SS.', ['3'], ''),
1118 (f' {tc}4{ts}: in the format 1w2d3h4m5s.', ['4'], ''),
1119 (f' {tc}5{ts}: as a verbose duration string.', ['5'], '') ]
1120 sb_overwrite = f'{cu+gr}Search results will overwrite the current subset.{ts}'
1121 options = [ (f' {tc+bd}A) {ts+tc}CURRENT SUBSET{ts}', 'A'),
1122 (f' {tc}1{ts}: Define a new subset of Qmap IDs.', ['1'], 'A'),
1123 (f' {tc}2{ts}: Display the current subset of {bw}' + ' {} ' + f'{ts} Qmap IDs.', ['2'], 'A'),
1124 (f' {tc}3{ts}: Display (a field of) all Qposts from the current subset.', ['3'], 'A'),
1125 (f' {tc}4{ts}: Display unique values from a field of all Qposts from the current subset.', ['4'], 'A'),
1126 (f' {tc}5{ts}: Display a list of formatted datetimes of all Qposts from the current subset.', ['5'], 'A'),
1127 (f' {tc}6{ts}: Display a list of relative time-intervals between the Qposts from the current subset.', ['6'], 'A'),
1128 (f' {tc}7{ts}: Display a list of character/word-frequencies for Qposts from the current subset.', ['7'], 'A'),
1129 (f' {tc}8{ts}: Find matching Regular Expression groups in the Qposts from the current subset.', ['8'], 'A'),
1130 (f' {tc}9{ts}: Find the longest common substring in (a field of) all Qposts from the current subset.', ['9'], 'A'),
1131 (f' {tc+bd}B) {ts+tc}SEARCH ALL QPOSTS{ts}', 'B'),
1132 (f'{tc}10{ts}: Case-sensitive text search in (a field of) all {bw}' + ' {} ' + f'{ts} Qposts; {sb_overwrite}', ['10'], 'B'),
1133 (f'{tc}11{ts}: Find all Qposts matching a Regular Expression; {sb_overwrite}', ['11'], 'B'),
1134 (f'{tc}12{ts}: Date/Time search in all Qposts; {sb_overwrite}', ['12'], 'B'),
1135 (f'{tc}13{ts}: Find all (earlier) Q-Clock aligned Qposts for a given date; {sb_overwrite}', ['13'], 'B'),
1136 (f'{tc}14{ts}: Find all (earlier) Q-Clock aligned Qposts for a given Qmap ID; {sb_overwrite}', ['14'], 'B'),
1137 (f'{tc}15{ts}: Find all Q-Clock aligned Qposts for a given clock time (HH:MM); {sb_overwrite}', ['15'], 'B'),
1138 (f' {tc+bd}C) {ts+tc}DOWNLOAD / SAVE{ts}', 'C'),
1139 (f'{tc}16{ts}: Check online if there are new Qposts available.', ['16'], 'C'),
1140 (f'{tc}17{ts}: Download the latest Qposts JSON-file from qanon.pub to your Downloads folder.', ['17'], 'C'),
1141 (f'{tc}18{ts}: Export all (new) Qposts to an SQLite3 Database inside your Downloads folder.', ['18'], 'C'),
1142 (f'{tc}19{ts}: Download images from the media/image URLs from all Qposts in the current subset.', ['19'], 'C') ]
1143 db_options = [ (f' {tc+bd}D) {ts+tc}SQLITE3 QPOSTS DATABASE{ts}', 'D'),
1144 (f'{tc}20{ts}: Display the results of an SQL SELECT-query from your Qposts Database.', ['20'], 'D') ]
1145 message = [ f'\n{tc+iv+cu}Qposts research tools menu:{ts}',
1146 f'{mc}Please choose one of the menu options ( or q to quit ):{ts} ',
1147 f'{tc}Type one of the commands: ' +
1148 f_all( ['all','first N','last N','prev N','next N','reverse','sort up','sort down'], f'{mc+iv}', f'{ts}', ' ' ) +
1149 f"{tc},{ts}\n{tc}or type a series of Qmap IDs and/or Qmap ID subranges separated by comma's,{ts}\n" +
1150 f"{tc}where Qmap ID subranges can be specified by placing a dash '{mc+iv}-{ts+tc}' in between two Qmap IDs,{ts}\n" +
1151 f"{tc}and where an asterisk '{mc+iv}*{ts+tc}' represents the most recent Qmap ID, for example:{ts+gr} 5-8,2777-*{ts}\n" +
1152 f"{mc}Please enter a new subset of Qmap IDs:{ts} ",
1153 f"{er}Incorrect list format: should only be integers separated by comma's, for example:{ts} 1,2,82,14",
1154 f'{er}Incorrect Qmap ID: the numbers should be from 1 to {n_posts}.{ts}',
1155 f'{er}Incorrect range format: should be 2 integers separated by a hyphen, for example:{ts} 1-10',
1156 f'{tc+cu}Fields:{ts} ' + f_all( _QPOST_KEYS, f'{mc+iv}', f'{ts}', ' ' ) +
1157 f'{ts}\n{mc}Please enter a field name/number (or nothing for the whole record):{ts} ',
1158 f'{tc+cu}Current subset of Qmap IDs:{ts}',
1159 f'{tc}Search terms can be combined using the keywords ' + f_all( ['and','or','not'], f'{mc+iv}', f'{tc}') +
1160 f' and using {mc+iv}(parentheses){tc}.{ts}\n{tc}Atoms containing spaces, keywords or parentheses ' +
1161 f'should be enclosed in {mc+iv}"double quotation marks"{tc}.{ts}\n' +
1162 f'{mc}Please enter a Case-sensitive search term:{ts} ',
1163 f'{tc}Date search terms must have the format: "OP DATETIME" (including quotation marks),{ts}\n' +
1164 f'{tc}where OP is one of the comparison operators ' + f_all( ['on','>=','<=','!=','=','>','<'], f'{mc+iv}', f'{tc}' ) +
1165 f',{ts}\n{tc}and where DATETIME is either a timestamp or a verbose date string like "28 Oct 2017".{ts}\n' +
1166 f'{tc}Both parts "OP DATETIME" together must be enclosed in {mc+iv}"double quotation marks"{tc}.{ts}\n' +
1167 f'{tc}Date Search terms can be combined using the keywords ' + f_all( ['and','or','not'], f'{mc+iv}', f'{tc}') +
1168 f', and using {mc+iv}(parentheses){tc}.{ts}\n{tc}The {mc+iv}on{tc} operator selects all posts from the same day, ' +
1169 f'for example:{ts} \"on 11 nov 2019\"\n{mc}Please enter a date search term:{ts} ',
1170 f'{tc+cu}Relative durations between Qposts:{ts}',
1171 f'{tc+cu}Formatted posting datetimes:{ts}',
1172 f'{tc}Interval Durations can be represented in one of the following formats:{ts}',
1173 f'{mc}Please enter a date after 28 October 2017:{ts} ',
1174 f'{mc}Please enter a valid Qmap ID number:{ts} ',
1175 f'{cu+gr} ? See ' + ansi_hyperlink(_DOC_STRFTIME,"strftime format codes",style=(0,1,1,0,0)) + f':\n' +
1176 f' {mc+iv}%d{tc} = day number (01 to 31); {mc+iv}%U{tc} = week number (00 to 53); {ts}\n' +
1177 f' {mc+iv}%A{tc} = weekday; {mc+iv}%a{tc} = weekday abbr.; {mc+iv}%w{tc} = weekday number (0 to 6); {ts}\n' +
1178 f' {mc+iv}%B{tc} = month; {mc+iv}%b{tc} = month abbr.; {mc+iv}%m{tc} = month number (01 to 12); {ts}\n' +
1179 f' {mc+iv}%y{tc} = year number (00 to 99); {mc+iv}%Y{tc} = year number (0000 to 9999);{ts}\n' +
1180 f' {mc+iv}%H{tc} = hour (00 to 23); {mc+iv}%I{tc} = hour (01 to 12); {mc+iv}%p{tc} = "AM" or "PM";{ts}\n' +
1181 f' {mc+iv}%M{tc} = minute (00 to 59); {mc+iv}%S{tc} = second (00 to 59); {mc+iv}%s{tc} = timestamp; {ts}\n' +
1182 f' {mc+iv}%c{tc} = datetime; {mc+iv}%x{tc} = date; {mc+iv}%X{tc} = time. {ts}\n' +
1183 f'{tc} {cu}The default format string is a long datetime: {mc+iv}{_DTM_FORMAT}{ts}\n' +
1184 f'{mc}Please enter a valid strftime format string (or nothing for default):{ts} ',
1185 f'{mc}Please enter a digital clocktime Hour and Minute (H:M):{ts} ',
1186 f'{tc}NB. at' + ' {} digital time, the Hour-hand on the analogue Qclock points to the {}-minute mark{},' +
1187 f'{ts}\n{tc}and forms a' + ' {}-degree angle with the Minute-hand.' + f'{ts}',
1188 f'{tc}The symbol {mc+iv}^{tc} matches the start of the text, {mc+iv}${tc} matches the end of the text, and {mc+iv}\\n{tc} ' +
1189 f'matches a newline character.{ts}\n{tc}Matched groups can be captured within parentheses, e.g.{ts} These people are (.*)\\n\n' +
1190 f'{mc}Please enter a Regular Expression ( capturing groups ):{ts} ',
1191 f'{mc}Please enter a Regular Expression pattern to find in all Qposts:{ts} ',
1192 f'{mc}Enter the lexical element to count: 0 = characters, or 1 = words:{ts} ',
1193 f'{mc}Also download images from referenced posts? [Y/n]:{ts} ',
1194 f'{tc} Database :{ts} ' + ansi_hyperlink( f'file://{_URL_QPOSTS_DB}', _SAFE_URL_QPOSTS_DB, fg=(150,190,10) ) + '\n' +
1195 f'{tc} Table name :{ts} {mc+iv}qposts{ts}\n' +
1196 f'{tc} Field names :{ts} ' + f_all( _QPOST_DB_COLS, f'{mc+iv}', f'{ts}', ' ') + f'\n' +
1197 f'{tc} Query Format:{ts} {mf}SELECT {cu}<fields>{mf} FROM {cu}qposts{mf} WHERE {cu}<condition>{mf};{ts}\n' +
1198 f"{tc} For Example :{ts} SELECT qmap_id, text FROM qposts WHERE strftime( '%w/%m', timestamp, 'unixepoch')='4/05';\n" +
1199 f"{' '*15}{cu+gr}( This example above finds all Qposts that are posted on a Friday in May ){ts}\n" +
1200 f"{' '*15}{cu+gr}? See " + ansi_hyperlink(_DOC_SQLITE_SLCT,'SQLite Select',style=(0,1,1,0,0)) +
1201 f'{cu+gr} (online) for more information about SQLite SELECT queries.{ts}\n' +
1202 f"{' '*15}{cu+gr}? See " + ansi_hyperlink(_DOC_SQLITE_FUNC,'SQL Functions',style=(0,1,1,0,0)) +
1203 f'{cu+gr} (online) for more functions that can be used inside the query.{ts}\n' +
1204 f'{mc}Please enter a valid SQL SELECT query:{ts} ',
1205 f'{er}Incorrect SELECT Query: must start with the word SELECT followed by a space and a value.{ts}' ]
1206 def input_qpost_key( default='' ):
1207 '''Ask user to enter a valid qpost field key; else it returns the given default.'''
1208 k, answer = 0, input( message[6] )
1209 if answer.isdigit(): k = int( answer )
1210 return _QPOST_KEYS[k-1] if k > 0 and k <= len( _QPOST_KEYS ) else ( answer if answer in _QPOST_KEYS else default )
1211 def handle_search_results( tuples, key, print_tuples=True ):
1212 '''Display the search results specified by <tuples> and <key>; returns the new subset of Qmap IDs.'''
1213 qmap_ids = [ qpt[0] for qpt in tuples ] if tuples else [] # construct Current Subset.
1214 if print_tuples: print_qpost_tuples( tuples, key )
1215 print( message[7], integer_list_to_range_string( qmap_ids ) )
1216 return qmap_ids
1217 while choice not in ['_EXIT','_ERROR']:
1218 menu = options.copy()
1219 menu[2] = (options[2][0].format( len(_QMAP_IDS) ), options[2][1], options[2][2]) # insert current subset size.
1220 menu[11] = (options[11][0].format( len(_QPOSTS) ), options[11][1], options[11][2]) # insert current number of Qposts.
1221 if os.path.exists( _URL_QPOSTS_DB ): menu.extend( db_options )
1222 choice = input_menu( menu, message[0], message[1], visible=_VISIBLE_MENU_GROUPS )
1223 if choice in menu[1][1]: #(1) DEFINE A SUBSET OF QMAP IDS:
1224 range_string = input( message[2] )
1225 rs_lower = range_string.lower()
1226 if rs_lower in [ 'reverse', 'rev' ]: _QMAP_IDS.reverse()
1227 elif rs_lower in [ 'sort down', 'sort desc', 'desc' ]: _QMAP_IDS.sort( reverse=True )
1228 elif rs_lower in [ 'sort up', 'sort asc', 'sort', 'asc' ]: _QMAP_IDS.sort()
1229 else:
1230 range_string = description_to_range_string( range_string, len(_QPOSTS), _QMAP_IDS )
1231 _QMAP_IDS = range_string_to_integer_list( range_string, validate=is_valid_qmap_id, msgs=message[3:6] )
1232 print( message[7], integer_list_to_range_string( _QMAP_IDS ) )
1233 elif choice in menu[2][1]: #(2) DISPLAY CURRENT SUBSET OF QMAP IDS:
1234 print( message[7], integer_list_to_range_string( _QMAP_IDS ) )
1235 elif choice in menu[3][1]: #(3) DISPLAY QPOSTS (or FIELDS) FROM THE CURRENT SUBSET:
1236 answer = input_qpost_key()
1237 tuples = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
1238 print_qpost_tuples( tuples, answer )
1239 elif choice in menu[4][1]: #(4) DISPLAY UNIQUE VALUES FROM A FIELD IN THE QPOSTS FROM THE CURRENT SUBSET:
1240 answer = input_qpost_key( 'trip' )
1241 tuples = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
1242 print_unique_qpost_field_values( tuples, answer )
1243 elif choice in menu[5][1]: #(5) DISPLAY FORMATTED POSTING DATETIMES OF QPOSTS FROM THE CURRENT SUBSET:
1244 strftime_format = input( message[15] )
1245 if strftime_format == '': strftime_format = _DTM_FORMAT
1246 dates = get_qpost_dates_for_qmap_ids( _QMAP_IDS )
1247 print( message[11], [ d.strftime( strftime_format ) for d in dates ] )
1248 elif choice in menu[6][1]: #(6) DISPLAY RELATIVE TIME INTERVALS BETWEEN QPOSTS FROM THE CURRENT SUBSET:
1249 tuples = get_qposts_by_id( _QMAP_IDS, key='timestamp', qmap_ids=True )
1250 i_option = input_menu( duration_options, message[12], message[1] )
1251 previous, durations = 0, []
1252 for i,tstamp in enumerate( tuples ):
1253 if i == 0: previous = tstamp[1]; durations.append( ( tstamp[0], 0 ) )
1254 else:
1255 difference = tstamp[1] - previous
1256 if i_option == '1' : durations.append( ( tstamp[0], difference ) )
1257 elif i_option in ['2','3'] : durations.append( ( tstamp[0], seconds_to_HMS( difference, days=(i_option == 3) ) ) )
1258 elif i_option == '4' : durations.append( ( tstamp[0], seconds_to_timestring( difference, separator='' ) ) )
1259 elif i_option == '5' : durations.append( ( tstamp[0], seconds_to_timestring( difference, [], True, ', ' ) ) )
1260 previous = tstamp[1]
1261 print( message[10], durations )
1262 elif choice in menu[7][1]: #(7) DISPLAY CHAR/WORD FREQUENCIES IN THE CURRENT SUBSET:
1263 lex = input( message[20] )
1264 lex = 0 if lex not in ['0','1'] else int( lex )
1265 answer = input_qpost_key( 'text' )
1266 tuples = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
1267 print_qpost_field_frequency_list( tuples, key=answer, lex=lex )
1268 elif choice in menu[8][1]: #(8) DISPLAY MATCHING REGEXP GROUPS IN THE CURRENT SUBSET:
1269 regexp = input( message[18] )
1270 answer = input_qpost_key()
1271 tuples = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
1272 ret = find_regexp_groups( tuples, fr'{regexp}' )
1273 print( f'{tc}Matched Groups for key=\'{answer}\':{ts} ', ret )
1274 elif choice in menu[9][1]: #(9) FIND LONGEST COMMON SUBSTRING IN THE CURRENT SUBSET:
1275 answer = input_qpost_key()
1276 tuples = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
1277 str_lcs = longest_common_substring( [ str(qpt[1]) for qpt in tuples ] )
1278 print( f'{tc}Longest Common Substring for key=\'{answer}\':{ts} "{str_lcs}"' )
1279 elif choice in menu[11][1]: #(10) SEARCH SUBSTRING IN ALL QPOSTS, CREATING A NEW SUBSET:
1280 search_term = input( message[8] )
1281 answer = input_qpost_key()
1282 tuples = search_qposts( search_term, key=answer )
1283 print( f'{tc}Found {bw} {len(tuples)} {tc} Qposts matching search term="{search_term}".{ts}' )
1284 _QMAP_IDS = handle_search_results( tuples, answer )
1285 elif choice in menu[12][1]: #(11) FIND QPOSTS MATCHING A REGEXP, CREATING A NEW SUBSET:
1286 search_term_regexp = input( message[19] )
1287 answer = input_qpost_key()
1288 tuples = search_qposts_regexp( search_term_regexp, key=answer )
1289 print( f'{tc}Found {bw} {len(tuples)} {tc} Qposts matching regexp="{search_term_regexp}".{ts}' )
1290 _QMAP_IDS = handle_search_results( tuples, answer )
1291 elif choice in menu[13][1]: #(12) DATE SEARCH IN ALL QPOSTS, CREATING A NEW SUBSET:
1292 date_condition = input( message[9] )
1293 answer = input_qpost_key()
1294 tuples = search_qposts_by_date( date_condition )
1295 print( f'{tc}Found {bw} {len(tuples)} {tc} Qposts matching date condition="{date_condition}".{ts}' )
1296 _QMAP_IDS = handle_search_results( tuples, answer )
1297 elif choice in menu[14][1]: #(13) SEARCH Q-CLOCK ALIGNED QPOSTS FOR DATE, CREATING A NEW SUBSET:
1298 date_string = input( message[13] )
1299 if not date_string: date_string = 'today'
1300 tuples = qclock_get_aligned_qposts_for_date( date_string )
1301 print( f'{tc}Found {bw} {len(tuples)} {tc} aligned Qposts for date={date_string}.{ts}' )
1302 _QMAP_IDS = handle_search_results( tuples, '', False )
1303 elif choice in menu[15][1]: #(14) SEARCH Q-CLOCK ALIGNED QPOSTS FOR QMAP ID, CREATING A NEW SUBSET:
1304 qmap_id = input( message[14] )
1305 if qmap_id.isdigit() and is_valid_qmap_id( int( qmap_id ) ): qmap_id = int( qmap_id )
1306 else: qmap_id = n_posts
1307 tuples = qclock_get_aligned_qposts_for_qmap_id( qmap_id )
1308 print( f'{tc}Found {bw} {len(tuples)} {tc} aligned Qposts for Qmap ID={qmap_id}.{ts}' )
1309 _QMAP_IDS = handle_search_results( tuples, '', False )
1310 elif choice in menu[16][1]: #(15) SEARCH Q-CLOCK ALIGNED QPOSTS FOR CLOCKTIME, CREATING A NEW SUBSET:
1311 s_time = input( message[16] )
1312 s_parts = s_time.split()
1313 if len( s_parts ) < 2: s_parts = s_time.split( ':' )
1314 if len( s_parts ) < 2: s_parts = s_time.split( ',' )
1315 try: qtime = ( int( s_parts[0] ), int( s_parts[1] ) )
1316 except: qtime = (4,20)
1317 qtime = ( ( qtime[0] + qtime[1] // 60 ) % 12, qtime[1] % 60 ) # normalize clocktime.
1318 tuples = qclock_get_aligned_qposts_for_clocktime( qtime, _QCLOCK_ROUND_NEAREST )
1319 hands_angle = clocktime_to_angle( qtime[0], qtime[1] )
1320 hr_pos = clocktime_hourhand_pos( qtime[0], qtime[1] )
1321 hr_pos_rounded = f'{[int,round][_QCLOCK_ROUND_NEAREST](hr_pos)}'
1322 hr_pos_info = f'' if hr_pos == float( hr_pos_rounded ) else f' (rounded off from {hr_pos:0.2f})'
1323 sqtime = f'{qtime[0]:02d}:{qtime[1]:02d}'
1324 print( message[17].format( sqtime, hr_pos_rounded, hr_pos_info, f'{hands_angle:0.2f}' ) )
1325 print( f'{tc}Found {bw} {len(tuples)} {tc} aligned Qposts for Qclocktime {bw} ({sqtime}) {tc}.{ts}' )
1326 _QMAP_IDS = handle_search_results( tuples, '', False )
1327 elif choice in menu[18][1]: #(16) CHECK FOR NEW QPOSTS:
1328 n_posts = check_update_local_qposts()
1329 elif choice in menu[19][1]: #(17) DOWNLOAD QPOSTS JSON-FILE FROM QANON.PUB:
1330 qposts = download_qposts_json( _URL_QPOSTS )
1331 if qposts:
1332 _QPOSTS = qposts; n_posts = len(_QPOSTS)
1333 print( f'{tc}File "Qposts.json" with {bw} {n_posts} {tc} Qposts successfully downloaded and saved to your Downloads folder.{ts}' )
1334 elif choice in menu[20][1]: #(18) EXPORT QPOSTS TO SQLITE3 DATABASE:
1335 qposts_to_sqlite( _QPOSTS, _URL_QPOSTS_DB )
1336 elif choice in menu[21][1]: #(19) DOWNLOAD IMAGES FROM THE CURRENT SUBSET:
1337 answer = input( message[21] )
1338 download_qpost_images( _QMAP_IDS, answer.lower() not in ['n','no'] )
1339 elif choice in menu[23][1]: #(20) PERFORM SQL SELECT QUERY IN QPOST.DB:
1340 sql = input( message[22] )
1341 if sql.lower().startswith( 'select ' ):
1342 if not sql.endswith( ';' ): sql += ';'
1343 res = qposts_sqlite_query( _URL_QPOSTS_DB, sql )
1344 print( f'{tc}SQL Query Result:{ts} ', res )
1345 else: print( message[23] )
1346 else: pass
1347 ##### END of qposts_terminal_loop()
1348
1349
1350#################
1351# Auxiliary Methods:
1352
1353def input_menu( menu, header='', prompt='> ', invalid='[1;47;31mInvalid choice.[0m', quits=['q','quit','exit'], visible={} ):
1354 '''Ask user to input a choice from a menu.
1355 Menuitems can be displayed in groups, which can be individually collapsed or expanded by entering the group key.
1356 To collapse or expand all groups at once, the user can enter the builtin commands HIDE or SHOW respectively.
1357 NB. Menuitems that are currently hidden, are not valid choices; First the menuitem must be made visible before it can be chosen.
1358 The function returns the key of the chosen menuitem, or '_EXIT' if the user chose to quit.
1359 <menu> : An ordered list of tuples, each with either 2 or 3 elements:
1360 For a group-header item, pass a 2-tuple( str, str ) where the first element is the header text to be displayed
1361 (including its key), and the second element is a key which identifies a group of menuitems.
1362 For a choosable menu item, pass a 3-tuple( str, list, str ) where the first element is the displayed text for this menuitem
1363 (including its preferred key), the 2nd element is a list of keys that the user can enter to select this particular choice,
1364 and the 3rd element is the key of the group that this menuitem belongs to ( i.e. the 2nd element of a group-header item ).
1365 <header> : string to be displayed before the list of menu choices; Leave empty if you don't want a menu header to be displayed.
1366 <prompt> : string to be displayed after the list of menu choices; This string should be asking for user input.
1367 <invalid> : string to be displayed when the user input is not recognized.
1368 <quits> : list of lowercase string commands that will exit the input loop (when typed in any case).
1369 <visible> : dict of {group:int} defining the initial visibility for each group; Updated in-place when user changes settings.'''
1370 _COLLAPSE = ['hide', 'collapse', 'none'] # Commands to collapse all menu groups.
1371 _EXPAND = ['show', 'expand', 'all'] # Commands to expand all menu groups.
1372 def print_menu():
1373 m_keys = []
1374 if header: print( header )
1375 for item in menu: # print visible menuitems and construct a list of valid keys:
1376 if isinstance( item, tuple ) and len( item ) >= 2:
1377 if len( item ) == 2: print( item[0] )
1378 elif visible.get( item[2], 1 ): m_keys.extend( item[1] ); print( item[0] )
1379 return m_keys
1380 ch, menuitem_keys, group_keys = '', [], []
1381 for item in menu: # Populate visibility dict and list of group keys:
1382 if isinstance( item, tuple ) and len( item ) == 2:
1383 if isinstance( item[1], str ):
1384 if item[1] not in visible: visible[ item[1] ] = 1
1385 group_keys.append( item[1] )
1386 menuitem_keys = print_menu() # Display the menu:
1387 while ch not in menuitem_keys:
1388 ch = input( prompt )
1389 if ch.lower() in quits: return '_EXIT' # User chose to Quit.
1390 if ch.lower() in _COLLAPSE: # Hide all menu groups.
1391 for gv in visible: visible[ gv ] = 0
1392 menuitem_keys = print_menu()
1393 elif ch.lower() in _EXPAND: # Show all menu groups.
1394 for gv in visible: visible[ gv ] = 1
1395 menuitem_keys = print_menu()
1396 elif ch in group_keys: # Hide/Show a specific menu group.
1397 visible[ ch ] = 1 - visible[ ch ]
1398 menuitem_keys = print_menu()
1399 elif ch in menuitem_keys: return ch # User chose a menu option.
1400 elif invalid: print( invalid )
1401 return '_ERROR'
1402
1403
1404def ansi_hyperlink( uri, text='Ctrl-Click Here', fg=None, bg=None, style=None ):
1405 '''Returns a string that can be displayed as a Ctrl-clickable hyperlink in the terminal.
1406 User can also right-click on the link to popup a contextmenu with options 'Open Hyperlink' and 'Copy Hyperlink Address'.
1407 Hovering the mouse over the hyperlink will popup a Tooltip showing the target uri.
1408 Hyperlink targets are opened using the system's default application for the target type.
1409 <uri> should be an urlencoded string containing only ascii characters 32 to 126, starting with an uri scheme identifier.
1410 Supported uri schemes a.o.: "http://", "https://", "ftp://", "file://", "mailto:".
1411 <text>: String representing the Ctrl-clickable text to be displayed.
1412 <fg>: None, or a 3-tuple of integers representing the RGB-values of the Foreground Color for <text>.
1413 <bg>: None, or a 3-tuple of integers representing the RGB-values of the Background Color for <text>.
1414 <style>: None, or a 5-tuple of Booleans for displaying <text> in Bold, Italic, Underline, Strikethrough, Blink.
1415 NB. if you pass <fg>, <bg>, and/or <style>, then any existing formatting of the text before the link will not be continued
1416 after the link. In that case, pass non-destructive ansi-code via <text> itself.'''
1417 if not (fg or bg or style): return fr"]8;;{uri}\\{text}]8;;\\"
1418 scv = [('','1;'),('','3;'),('','4;'),('','9;'),('','6;')]
1419 sbg = f"48;2;{bg[0]};{bg[1]};{bg[2]}" if bg and len( bg ) >= 3 else ''
1420 sfg = f"{';' if sbg else ''}38;2;{fg[0]};{fg[1]};{fg[2]}" if fg and len( fg ) >= 3 else ''
1421 stl = ''.join( [scv[i][bool(s)] for i,s in enumerate( style )] ) if style and len( style ) == 5 else ''
1422 stl = stl if sbg or sfg else stl[:-1]
1423 rst = '[0m' if stl or sbg or sfg else ''
1424 return f"]8;;{uri}\\[{stl}{sbg}{sfg}m{text}{rst}]8;;\\"
1425
1426
1427def collapse_user( str_path ): # inverse of os.path.expanduser()
1428 '''Replaces the user directory in a path by a tilde ( to hide the user name ).'''
1429 return str_path.replace( os.path.expanduser('~'), '~' )
1430
1431
1432def download_binary_file( url, save_url ):
1433 '''Tries to download a file from the Internet, and save it into the specified location <save_url>.
1434 Does NOT check if the file type indicated in <save_url> is the same as the file type from <url>.
1435 This function returns True if the download succeeded, else it returns False.'''
1436 def printq( msg ): print( msg ); return False
1437 response, headers = None, {'User-Agent': 'Mozilla/5.0'}
1438 try: response = urlopen( Request( url, headers=headers) ) # from urllib.request import Request, urlopen
1439 except URLError as e: # from urllib.error import URLError
1440 return False; printq( f"{er}URLError; cannot download file '{url}.'\tReason: {e.reason}.{ts}" )
1441 except: printq( f"{er}Error downloading file '{url}'.{ts}" )
1442 if not response: return False
1443 elif response.status != 200:
1444 printq( f"{er}Download Failure: Response has status code {response.status}.{ts}" )
1445 else:
1446 try:
1447 with open( save_url, 'wb' ) as f: f.write( response.read() )
1448 return True
1449 except: pass # ConnectionResetError
1450 return False
1451
1452
1453def parse_datetime_string( datetime_string='now', format_as='%c', parserinfo=None ):
1454 '''Parses a datetime string such as "Sept 17th, 1984 at 01:30 AM", and returns a 3-tuple containing the parsed datetime,
1455 a string expressing the parsed datetime in the specified format, and a rest tuple of unparsed tokens.
1456 If the parsing failed, this function returns (None,'','').
1457 <datetime_string>: String specifying a datetime to be parsed; The string can also specify a timestamp.
1458 <format_as>: determines the format of the datetime string to be returned; Default "%c" is locale date&time format.
1459 if the format is '' or None, a floating point timestamp will be returned instead of a string.
1460 NB. Uses the module dateutil; results are not so good for verbose date strings inside a sentence.'''
1461 #import dateutil.parser as dateparser #import datetime
1462 dts = datetime_string.strip() if isinstance( datetime_string, str ) else 'now'
1463 if dts.lower() in [ 'now', 'today' ]: dt, rest = datetime.datetime.now(), ()
1464 elif is_numeral( dts ): dt, rest = datetime.datetime.fromtimestamp( float( dts ), tz=None ), () # interpret input as timestamp.
1465 else:
1466 try: dt, rest = dateparser.parse( dts, parserinfo=parserinfo, default=None, dayfirst=True, yearfirst=False,
1467 ignoretz=True, fuzzy_with_tokens=True )
1468 except ValueError: return (None,'','') # input could not be parsed. (Pass a custom parserinfo for the local user language?).
1469 except: return (None,'','') # OverflowError?: parsed date exceeds the largest valid C integer.
1470 if dt: return dt, dt.strftime( format_as ) if format_as else dt.timestamp(), rest
1471 return (None,'','')
1472
1473
1474def seconds_to_timestring( seconds, units=['w','d','h','m','s'], add_s=False, separator=' ' ):
1475 '''Turns a number of seconds into a human-readable duration string, expressed in weeks, days, hours, minutes, and seconds.
1476 <seconds>: The total number of seconds in the duration; Can pass a float, but the decimal part is not used.
1477 <units> : Symbols for each of the 5 durations (week, day, hour, minute, second), or <None> to use the verbose English words.
1478 <add_s> : If True, the character 's' will be appended after the unit symbol, if the number for that unit is larger than 1.
1479 <separator>: String to put in between each of the number/unit pairs.
1480 Adapted from function elapsed_time(): http://snipplr.com/view/5713/python-elapsedtime-human-readable-time-span-given-total-seconds/'''
1481 assert( isinstance( seconds, (int,float) ) )
1482 if not units or len(units) < 5: units = [' week',' day',' hour',' minute',' second'] # NB. space before the units.
1483 if seconds == 0: return '%s%s' % ( '0', units[-1] + ( '', 's' )[add_s] )
1484 if seconds < 0: return '-' + seconds_to_timestring( -seconds, units, add_s, separator )
1485 duration, lengths = [], [ 604800, 86400, 3600, 60, 1 ]
1486 for unit, length in zip( units, lengths ):
1487 value = seconds // length
1488 if value >= 1:
1489 seconds %= length
1490 duration.append( '%s%s' % ( str(value), (unit, (unit, unit + 's')[value > 1])[add_s] ) )
1491 if seconds < 1: break
1492 return separator.join( duration )
1493
1494
1495def seconds_to_HMS( seconds, microseconds=False, days=True ):
1496 '''Converts <seconds> into a string of the format "HH:MM:SS" with optional microseconds and/or days.'''
1497 assert( isinstance( seconds, (int,float) ) and isinstance( microseconds, bool ) and isinstance( days, bool ) )
1498 if seconds == 0: return '00:00:00' + ('.000000' if microseconds else '')
1499 if seconds < 0: return '-' + seconds_to_HMS( -seconds )
1500 minutes, seconds = divmod( seconds, 60 )
1501 hours, minutes = divmod( minutes, 60 )
1502 msecs = f'{seconds:09.6f}' if microseconds else f'{int(seconds):02d}'
1503 if not days: return f'{int(hours):02d}:{int(minutes):02d}:{msecs}'
1504 ds, hours = divmod( hours, 24 )
1505 d = f"{ds} day{['s',''][bool(ds==1)]}, " if ds else ''
1506 return f'{d}{int(hours):02d}:{int(minutes):02d}:{msecs}'
1507
1508
1509def timestamp_day_bounds( timestamp ):
1510 '''Returns a 2-tuple with timestamps for the start & end of the day in which the specified <timestamp> falls.
1511 NB. The starting bound is Inclusive, the ending bound is Exclusive ( being the start of the next day ).'''
1512 ts_ordinal = datetime.datetime.fromtimestamp( timestamp, tz=None ).toordinal() # Serial Day Number.
1513 ts_start = datetime.datetime.fromordinal( ts_ordinal ).timestamp() # Timestamp Day-Start.
1514 return ts_start, ts_start + 86400
1515
1516
1517def clocktime_to_angle( hour, minute ):
1518 '''''Returns a float representing the angle between the hour and minute hands of an analog clock showing the specified time.'''
1519 H = int( hour ) % 12
1520 M = int( minute ) % 60
1521 return H * 30 - M * 6 + M / 2
1522
1523
1524def clocktime_hourhand_pos( hour, minute ):
1525 '''''Returns a float representing the position (0-59.916666) of the Hour-hand of an analog clock showing the specified time.'''
1526 H = int( hour ) % 12
1527 M = int( minute ) % 60
1528 return ( H + M / 60 ) * 5
1529
1530
1531def integer_list_to_range_string( integer_list, sep='-' ):
1532 '''Returns a string representation of the specified list of integers <integer_list>, where
1533 ranges of consecutive integers are compacted to only their start- and end, separated by <sep>.
1534 E.g. for input = [1,2,3,4,5,6,7,8,55] it returns the string "1-8,55".'''
1535 parts, previous, direction = [], None, 0
1536 for n in integer_list:
1537 if isinstance( n, int ) and previous is not None:
1538 if direction == 0:
1539 start = previous
1540 if n - previous == 1: direction = 1
1541 elif n - previous == -1: direction = -1
1542 else: parts.append( str( previous ) )
1543 elif n - previous != direction:
1544 direction = 0
1545 parts.append( str( start ) + sep + str( previous ) )
1546 previous = n
1547 parts.append( str( previous ) if direction == 0 else str( start ) + sep + str( previous ) )
1548 return ','.join( parts )
1549
1550
1551def range_string_to_integer_list( range_string, sep='-', validate=None, msgs=[] ):
1552 '''Returns a list of integers based on the specified <range_string>.
1553 <range_string>: Comma-separated string of: numbers and/or <sep>-separated number ranges.
1554 <sep>: Symbol separating the start and end of the number ranges inside <range_string>.
1555 <validate>: None, or Pass a validation function that should accept an integer and return a Boolean.
1556 <msgs>: List of 3 error messages in case the input is: Not a Number, Invalid Number, Invalid Range.'''
1557 def print_msg( i ):
1558 if msgs and len(msgs) > i: print( msgs[i] )
1559 intlist, items = [], range_string.split( ',' )
1560 for item in items:
1561 if sep in item: # separator: defines a range.
1562 range_ext = item.split( sep )[0:2]
1563 if range_ext[0].isdigit() and range_ext[1].isdigit():
1564 nm1, nm2 = int( range_ext[0] ), int( range_ext[1] )
1565 if not callable( validate ) or ( validate( nm1 ) and validate( nm2 ) ):
1566 subrange = list( range( nm1, nm2 - 1, -1 ) if nm2 < nm1 else range( nm1, nm2 + 1 ) )
1567 intlist.extend( subrange )
1568 else: print_msg( 1 ); break
1569 else: print_msg( 2 ); break
1570 elif item.isdigit():
1571 nm = int( item )
1572 if not callable( validate ) or validate( nm ): intlist.append( nm )
1573 else: print_msg( 1 ); break
1574 else: print_msg( 0 ); break
1575 return intlist
1576
1577
1578def description_to_range_string( desc, n_max, cur=[] ):
1579 '''Interpret commands: "all", "first N", "last N", "previous N", "next N", and asterisk shortcut: "*",
1580 into a range_string that can be converted by range_string_to_integer_list(). '''
1581 lcase, commands = desc.lower(), [ 'first', 'last', 'next', 'prev', 'previous', ]
1582 if lcase == 'all': return f'1-{n_max}'
1583 if any( lcase.startswith( cmd ) for cmd in commands ):
1584 parts = desc.split()
1585 amount = 1 if len(parts) < 2 or not parts[1].isdigit() else int( parts[1] )
1586 amount = min( max( 1, amount ), n_max )
1587 if lcase.startswith( 'first' ): return f'1-{amount}'
1588 if lcase.startswith( 'last' ): return f'{n_max}-{n_max-amount+1}'
1589 cmin, cmax = ( min( cur ), max( cur ) ) if cur else ( 1, n_max )
1590 if lcase.startswith( 'next' ): return f'{min( max( 1, cmax + 1 ), n_max )}-{min( max( 1, cmax + amount ), n_max )}'
1591 if lcase.startswith( 'prev' ): return f'{min( max( 1, cmin - 1 ), n_max )}-{min( max( 1, cmin - amount ), n_max )}'
1592 else: return desc.replace( '*', f'{n_max}' ) # asterisk * means the maximum value <n_max>.
1593
1594
1595def count_frequencies( text, lex=0 ):
1596 '''Returns a list of 2-tuples(str, int) containing the count of each lexical element in <text>.
1597 <lex>: Determines which lexical element to count: 0=count characters; 1=count words.'''
1598 # from collections import Counter
1599 if isinstance( text, ( list, tuple, dict ) ): text = str( text )
1600 if isinstance( text, str ):
1601 if lex == 0: return Counter( text ).most_common()
1602 elif lex == 1: return Counter( text.split() ).most_common()
1603 return []
1604
1605
1606def longest_common_substring( data ):
1607 ''' Finds the longest common substring from a list of strings.'''
1608 # From https://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python/2894073#2894073
1609 substr = ''
1610 if len( data ) == 1: return data[0] # only 1 element: return itself as longest string.
1611 if len( data ) > 1:
1612 d, n = data[0], len( data[0] )
1613 if n > 0:
1614 for i in range( n ):
1615 for j in range( n - i + 1 ):
1616 if j > len( substr ) and is_common_substring( d[i:i+j], data ):
1617 substr = d[i:i+j]
1618 return substr
1619
1620def is_common_substring( find, data ):
1621 '''Used by longest_common_substring().'''
1622 if len( data ) < 1 or len( find ) < 1: return False
1623 for i in range( len(data) ):
1624 if find not in data[i]: return False
1625 return True
1626
1627
1628def is_numeral( s ):
1629 '''Returns True if the input string <s> represents either an integer number (e.g. '2500'), a floating
1630 point number (e.g. '2500.0'), a number expressed in scientific notation (e.g. '2.5E3'), or 'NaN'.'''
1631 try: _ = float( s ); return True
1632 except : return False
1633
1634
1635def f_all( s_list, s_before='', s_after='', sep=', ' ):
1636 return sep.join( [ s_before + k + s_after for k in s_list ] ) if s_list else ''
1637
1638
1639def split_list( lst ):
1640 '''Returns two "flat" lists: the first containing all items from the first dimension of <lst>,
1641 the second containing all items from the second and further dimensions of <lst>.'''
1642 def flatten_list( lst, newlist=[] ):
1643 for item in lst:
1644 if isinstance( item, list ): flatten_list( item, newlist )
1645 else: newlist.append( item )
1646 dim_one, dim_rest = [], []
1647 for item in lst:
1648 if isinstance( item, list ): flatten_list( item, dim_rest )
1649 else: dim_one.append( item)
1650 return dim_one, dim_rest
1651
1652
1653#################
1654# Class Simple_Logic_Expression
1655
1656class Simple_Logic_Expression:
1657 '''Represents a simple logical expression such as 'A and B or not C'.
1658 Only supports the logical connectives And, Or, Not, and parentheses;
1659 Atoms containing spaces or parentheses should be enclosed in "(double) quotation marks".'''
1660
1661 def __init__( self, str_expression, parse_format=1, eval_func=[], eval_args=(), operators=[] ):
1662 '''<str_expression>: String containing the simple logic expression to be parsed, e.g: 'A and B or not C'.
1663 <operators> : List of 5 symbols for [And, Or, Not, Left Paren, Right Paren], that can be used in <str_expression>.
1664 <parse_format> : Determines the format of the parsed output:
1665 0=list (prefix) e.g: ['or', ['and', 'A', 'B'], ['not', 'C']];
1666 1=string (infix) e.g: "(A and B) or (not C)".
1667 <eval_func> : pass an evaluation function that returns a Boolean for each atom used in <str_expression>,
1668 or pass a list of atoms that are used in <str_expression>, and whose value is <True>.
1669 <eval_args> : optional tuple of arguments to pass on to the evaluation function <eval_func>.'''
1670 if not operators or len( operators ) < 5: operators = [ 'and', 'or', 'not', '(', ')' ]
1671 self.expression = str_expression
1672 self.eval_func = eval_func
1673 self.eval_args = eval_args
1674 self.format = min(max( 0, parse_format ), 1) # 0=list (prefix); 1=string (infix).
1675 self._OP_AND = operators[0] #'and'
1676 self._OP_OR = operators[1] #'or'
1677 self._OP_NOT = operators[2] #'not'
1678 self._PAR_L = operators[3] #'('
1679 self._PAR_R = operators[4] #')'
1680
1681 def parse( self, parse_format=None ):
1682 '''Parse the current expression, optionally overriding the current parse format.
1683 based on: https://www.howtobuildsoftware.com/index.php/how-do/gbu/string-algorithm-parsing-algorithm-
1684 to-add-implied-parentheses-in-boolean-expression'''
1685
1686 def stream_starts_with( stream, token ):
1687 return stream[0:len(token)] == list(token)
1688
1689 def pop( stream, token ):
1690 if stream_starts_with( stream, token ):
1691 del stream[0:len(token)]
1692 return True
1693 return False
1694
1695 def parse_primary( stream ):
1696 if pop( stream, '"' ): return parse_enclosure( stream, '"', '"' ) # parse double quote.
1697 while pop( stream, ' ' ): pass
1698 if pop( stream, self._PAR_L ):
1699 e = parse_or( stream )
1700 pop( stream, self._PAR_R )
1701 return e
1702 return parse_atom( stream )
1703
1704 def parse_enclosure( stream, enc_left, enc_right ):
1705 r = [ '', enc_left][self.format] # keep/restore enclosure symbols if format=1.
1706 while stream and not pop( stream, enc_right ):
1707 r += stream.pop(0)
1708 while pop( stream, ' ' ): pass
1709 return r + [ '', enc_right][self.format]
1710
1711 def parse_binary( stream, operator, func ):
1712 while pop( stream, ' ' ): pass
1713 es = [func( stream )]
1714 while pop( stream, operator ): es.append( func( stream ) )
1715 if self.format == 0: return [operator, *es] if len(es) > 1 else es[0]
1716 else: return self._PAR_L + ' {} '.format(operator).join(es) + self._PAR_R if len(es) > 1 else es[0]
1717
1718 def parse_unary( stream ):
1719 while pop( stream, ' ' ): pass
1720 if pop( stream, self._OP_NOT ):
1721 if self.format == 0: return [ self._OP_NOT, parse_unary( stream ) ]
1722 else: return f'({self._OP_NOT} {parse_unary( stream )})'
1723 return parse_primary( stream )
1724
1725 def parse_or( stream ):
1726 while pop( stream, ' ' ): pass
1727 p = parse_binary( stream, self._OP_OR, parse_and )
1728 return p if p else parse_unary( stream )
1729
1730 def parse_and( stream ):
1731 while pop( stream, ' ' ): pass
1732 p = parse_binary( stream, self._OP_AND, parse_unary )
1733 return p if p else parse_unary( stream )
1734
1735 def parse_atom( stream ):
1736 atom = ''
1737 while stream and not pop( stream, ' ' ) and not stream_starts_with( stream, self._PAR_R ):
1738 atom += stream.pop(0)
1739 return atom
1740
1741 if parse_format is not None: self.format = parse_format
1742 #if not isinstance( self.expression, (list,str) ): return self.expression
1743 output = parse_or( list( self.expression ) )
1744 return output if self.format == 0 else output[1:-1] # Removes the outermost pair of parentheses.
1745
1746
1747 def evaluate( self, eval_func=None, eval_args=None ):
1748 '''Evaluate the current expression, optionally overriding the current atom evaluation function (or list).'''
1749
1750 def evaluate_atom( atom ):
1751 if isinstance( self.eval_func, list ): return atom in self.eval_func
1752 if callable( self.eval_func ): return self.eval_func( atom, *self.eval_args )
1753
1754 def evaluate_op( op_list ):
1755 operator = op_list[0]
1756 truthval = evaluate_output( op_list[1] )
1757 if operator == self._OP_NOT: return not truthval
1758 if operator == self._OP_AND:
1759 for arg in op_list[2:]: truthval = truthval and evaluate_output( arg )
1760 return truthval
1761 if operator == self._OP_OR:
1762 for arg in op_list[2:]: truthval = truthval or evaluate_output( arg )
1763 return truthval
1764 return truthval
1765
1766 def evaluate_output( output ):
1767 if isinstance( output, list ): return evaluate_op( output )
1768 if isinstance( output, str ): return evaluate_atom( output )
1769 return output
1770
1771 if eval_func is not None: self.eval_func = eval_func
1772 if eval_args is not None: self.eval_args = eval_args
1773 if not isinstance( self.expression, (list,str) ): return evaluate_atom( self.expression )
1774 output = self.parse( parse_format=0 ) # must be parse_format=0 (=Prefix list).
1775 return evaluate_output( output )
1776
1777
1778 def find_in( self, text ):
1779 '''Interprets the current expression as a complex search-string, and checks if it matches the target <text>.'''
1780 def eval_atom( atom ): return atom in text
1781 return self.evaluate( eval_atom )
1782
1783
1784 def match_value( self, value, eval_func ):
1785 '''<eval_func>: required evaluation function taking at least 2 arguments: the current atom and <value>.'''
1786 return self.evaluate( eval_func, (value,) )
1787
1788# End of Simple_Logic_Expression
1789
1790
1791def evaluate_comparison( value1, op, value2 ):
1792 if op.lower() in ['on']: # Returns True if value2 is a timestamp falling on the same calendar day as value1.
1793 ts_start, ts_end = timestamp_day_bounds( value2 )
1794 return value1 >= ts_start and value1 < ts_end
1795 elif op in ['>=']: return value1 >= value2
1796 elif op in ['<=']: return value1 <= value2
1797 elif op in ['!=']: return value1 != value2
1798 elif op in ['==', '=']: return value1 == value2
1799 elif op in ['>']: return value1 > value2
1800 elif op in ['<']: return value1 < value2
1801
1802
1803def evaluate_datetime_condition( dt_condition, timestamp ):
1804 '''Returns True if the expression "<timestamp><dt_condition>" evaluates as True.
1805 <dt_condition>: Numerical Timestamp, or a String of the format "COMP DATETIME" (including double quotation marks),
1806 where COMP is one of the comparison operators in [ 'on', '>=', '<=', '!=', '==', '=', '>', '<' ],
1807 and where DATETIME is a valid datetime expression, a timestamp, or 'now'.
1808 The 'on'-operator yields True iff the <timestamp> falls on the same day as the DATETIME specified in <dt_condition>.
1809 <timestamp>: Integer or Float to be compared against the DATETIME specified in <dt_condition>.'''
1810 if isinstance( dt_condition, (int,float) ): # interpret dt_condition as timestamp.
1811 return evaluate_comparison( timestamp, 'on', dt_condition )
1812 for op in [ 'on', '>=', '<=', '!=', '==', '=', '>', '<' ]:
1813 if dt_condition.startswith( op ):
1814 dt_string = dt_condition[len(op):].strip()
1815 if is_numeral( dt_string ): dts = float( dt_string ) # interpret dt_string as timestamp.
1816 else: dt, dts, rest = parse_datetime_string( dt_string, '' ) # slow; instead pass a timestamp for dt_condition.
1817 if dts: return evaluate_comparison( timestamp, op, dts )
1818 return False
1819
1820
1821def evaluate_numeric_condition( num_condition_string, num ):
1822 '''Returns True if the expression "<num><num_condition_string>" evaluates as True.
1823 <num_condition_string>: String of the format "COMP NUMBER" (including double quotation marks),
1824 where COMP is one of the comparison operators in [ '>=', '<=', '!=', '==', '=', '>', '<' ],
1825 and NUMBER is any number string that can be casted into a float, e.g.: '1', '2.2e3', etc.
1826 <num>: Integer or Float to be compared against the NUMBER specified in <num_condition_string>.'''
1827 for op in [ '>=', '<=', '!=', '==', '=', '>', '<' ]:
1828 if num_condition_string.startswith( op ):
1829 num_string = num_condition_string[len(op):]
1830 try: return evaluate_comparison( num, op, float( num_string ) )
1831 except: pass # Error if float() fails.
1832 return False
1833
1834
1835#################
1836# Main:
1837
1838if __name__ == '__main__':
1839
1840 # Read optional commandline parameters to override default settings:
1841 # Type "python3 qposts_research.py -h" to see a list of options.
1842 descr = 'Python script to facilitate research/analysis of Qposts.'
1843 epilog = 'Thank you for using %(prog)s ... WWG1WGA!'
1844 h_tips = 'Add this flag to hide tooltips for abbreviations.'
1845 h_wrap = 'Wrap width (in characters) for the Qposts Text field.'
1846 h_indn = 'Amount of indentation for nested references.'
1847 t_cols = ['black','red','green','yellow','blue','magenta','cyan','white']
1848 h_tcol = 'Label Color: ' + '; '.join([ f'{i}=[3{i}m{c}[0m' for i,c in enumerate(t_cols)]) + '.'
1849 h_menu = 'Initial visibility (0 or 1) for each menu group.'
1850 h_qset = 'Initial subset of Qmap IDs, e.g. "all" or "last 100".'
1851 h_date = 'Strftime format to display a short date string.'
1852 h_dtm = 'Strftime format to display a long date&time string.'
1853 h_vers = 'Show program version and exit.'
1854
1855 formatter = argparse.MetavarTypeHelpFormatter
1856 parser = argparse.ArgumentParser( description=descr, epilog=epilog, formatter_class=formatter )
1857 parser.add_argument( '-nt', '--notips', help=h_tips, default=(not _SHOW_ABBR_TOOLTIPS), action='store_true' )
1858 parser.add_argument( '-w', '--wrap', type=int, help=h_wrap, default=_MAX_WRAP )
1859 parser.add_argument( '-i', '--indent', type=int, help=h_indn, default=len( _LVL_INDENT ) )
1860 parser.add_argument( '-c', '--color', type=int, help=h_tcol, default=argparse.SUPPRESS )
1861 parser.add_argument( '-d', dest='date', type=str, help=h_date, default=_DT_FORMAT )
1862 parser.add_argument( '-dtm', dest='datetime', type=str, help=h_dtm, default=_DTM_FORMAT )
1863 parser.add_argument( '-m', dest='menu', type=int, help=h_menu, default=argparse.SUPPRESS, nargs='*' )
1864 parser.add_argument( '-s', dest='subset', type=str, help=h_qset, default=[_INITIAL_SUBSET], nargs='+' )
1865 parser.add_argument( '--version', action='version', help=h_vers, version='%(prog)s version 0.0.1b (20200117)' )
1866 args = parser.parse_args()
1867
1868 _INITIAL_SUBSET = ' '.join( args.subset )
1869 _SHOW_ABBR_TOOLTIPS = not args.notips
1870 _LVL_INDENT = ' ' * args.indent
1871 _MAX_WRAP = args.wrap
1872 _DT_FORMAT = args.date
1873 _DTM_FORMAT = args.datetime
1874 if hasattr( args, 'color' ):
1875 if args.color >= 0 and args.color <= 7:
1876 tc = f'[0;3{args.color}m'
1877 if hasattr( args, 'menu' ):
1878 mg_len = len( _VISIBLE_MENU_GROUPS )
1879 mg_keys = list( _VISIBLE_MENU_GROUPS.keys() )
1880 for i, vis in enumerate( args.menu ):
1881 if i < mg_len: _VISIBLE_MENU_GROUPS[mg_keys[i]] = int( bool( vis ) )
1882
1883
1884 print( f'{tc}{datetime.datetime.now().strftime( _DTM_FORMAT )}' )
1885
1886 _QPOSTS = open_qposts_json( _URL_QPOSTS ) # Open local Qposts.json file.
1887 if not _QPOSTS:
1888 print( f"{er}Local file Qposts.json not found:{ts} Attempting to download Qposts.json from qanon.pub ..." )
1889 _QPOSTS = download_qposts_json( _URL_QPOSTS )
1890
1891 if _QPOSTS:
1892 n_qposts = check_update_local_qposts() # Check for Latest Qposts online.
1893 print( f'{tc}Local Qposts.json: containing {bw} {n_qposts} {ts+tc} Qposts.{ts}' )
1894
1895 if os.path.exists( _URL_QPOSTS_DB ): # Check if SQLite3 QPosts.db exists:
1896 n_recs = qposts_sqlite_count_records( _URL_QPOSTS_DB )
1897 print( f'{tc}Local Qposts.db: containing {bw} {n_recs} {ts+tc} Qpost Records.{ts}' )
1898
1899 range_string = description_to_range_string( _INITIAL_SUBSET, n_qposts )
1900 _QMAP_IDS = range_string_to_integer_list( range_string ) # Create subset of Qmap_ID's.
1901 print( f'{tc+cu}Current subset of Qmap IDs:{ts}', integer_list_to_range_string( _QMAP_IDS ) )
1902
1903 qposts_terminal_loop() # Start terminal input loop.
1904 else: print( f"{er}No Qposts.json available:{tc} Exiting Program ...{ts}" )
1905
1906quit()
1907# End