74cHuSWc

1import os
2import datetime
3import utils.parsing as utils
4from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
5
6merge_map = { "rosaluxembourg@gmail.com": "rosa.luxembourg@protonmail.com" }
7
8def split_families(members):
9    family_accounts = [m for m in members if m['family_first_name']]
10    to_add = []
11    for f in family_accounts:
12        first_name = f['family_first_name'].strip("& ").replace(",\n","")
13        last_name = f['family_last_name'].strip("& ").replace(",\n","")
14        ak_id = f['ak_id'] + "f"
15
16        new_val = f.copy()
17        new_val['first_name'] = first_name
18        new_val['last_name'] = last_name
19        new_val['ak_id'] = ak_id
20        new_val['email'] = ""
21        to_add.append(new_val)
22    return to_add + members
23
24def merge_dupes(all_members):
25    #This is very inefficient
26    for email1, email2 in merge_map.items():
27        rows1 = [mem for mem in all_members if mem['email'] == email1]
28        rows2 = [mem for mem in all_members if mem['email'] == email2]
29
30        if rows1 and rows2:
31            join_date = min(rows1 + rows2, key=lambda x: x['join_date_parsed'])
32            x_date = min(rows1 + rows2, key=lambda x: x['xdate_parsed'])
33
34            for val in rows1:
35                val['join_date'] = join_date['join_date']
36                val['xdate'] = x_date['join_date']
37            for val in rows2:
38                all_members.remove(val)
39    return all_members
40
41def get_expired(all_members, date):
42    if date is None:
43        today = datetime.datetime.now()
44        date = [today.year, today.month, today.day]
45
46    expired = [m for m in all_members if m['xdate_parsed'] <= date]
47    migs = [m for m in all_members if m['xdate_parsed'] > date]
48    return expired, migs
49
50def same_date(old_date, date):
51    if not isinstance(old_date, str):
52        return False
53
54    splitted = old_date.split('/')
55    if len(splitted) != 3:
56        return False
57    if len(splitted[-1]) > 2:
58        splitted[-1] = splitted[-1][:-2]
59
60    splitted2 = old_date.split('/')
61    if len(splitted2) != 3:
62        return False
63    if len(splitted2[-1]) > 2:
64        splitted2[-1] = splitted2[-1][:-2]
65
66    return splitted2 == splitted
67
68def get_newly_expired(new_members, old_list, date=None, lookback=2):#months
69    if date is None:
70        today = datetime.datetime.now()
71        date = [today.year, today.month, today.day]
72    last_date = [date[0], date[1] - lookback, date[2]]
73
74    new = []
75    new_expired = []
76    missing = []
77    changes = []
78    renewed = []
79
80    old_map = {str(o['ak_id']): o for o in old_list}
81    found_map = {str(o['ak_id']): False for o in old_list}
82    for val in new_members:
83        if val['ak_id'] in old_map:
84            found_map[val['ak_id']] = True
85
86            if val['xdate_parsed'] > last_date and val['xdate_parsed'] <= date:
87                print(val['xdate_parsed'])
88                new_expired.append(val)
89
90            old_vals = old_map[val['ak_id']]
91            if old_vals['xdate_parsed'] < val['xdate_parsed'] and old_vals['xdate_parsed'] <= date:
92                renewed.append(val)
93
94            different = False
95            for o in old_vals:
96                if o in ['xdate_parsed', 'join_date_parsed']:
97                    continue
98                if o in val and old_vals[o] != val[o]:
99                    if not same_date(old_vals[o], val[o]):
100                        #print(o, old_vals[o], val[o])
101                        different = True
102            if different:
103                changes.append(val)
104        else:
105            new.append(val)
106
107    for ak_id,found in found_map.items():
108        if not found:
109            missing.append(old_map[ak_id])
110
111    return new_expired, new, missing, changes, renewed
112
113def main(new_list, headers, old_list, outdir="output", date=[2018, 12, 2]):
114    members = split_families(new_list)
115    members = list(merge_dupes(members))
116    expired, migs = get_expired(members, date)
117    new_expired, new, missing, changes, renewed = get_newly_expired(members, old_list, date)
118
119    if not os.path.exists(outdir):
120        os.makedirs(outdir)
121
122    #Write migs
123    utils.write_csv("{}/migs.csv".format(outdir), migs, headers)
124    print(len(migs), "migs")
125
126    #Write expired
127    utils.write_csv("{}/expired.csv".format(outdir), expired, headers)
128    print(len(expired), "expired")
129
130    #Write new
131    utils.write_csv("{}/new.csv".format(outdir), new, headers)
132    print(len(new), "new")
133
134    #Write newly expired
135    utils.write_csv("{}/new_expired.csv".format(outdir), new_expired, headers)
136    print(len(new_expired), "newly expired")
137
138    #Report missing
139    utils.write_csv("{}/missing.csv".format(outdir), missing, headers)
140    print(len(missing), "missing")
141
142    #Write changes
143    utils.write_csv("{}/changes.csv".format(outdir), changes, headers)
144    print(len(changes), "changes")
145
146    #Write renewed
147    utils.write_csv("{}/renewed.csv".format(outdir), renewed, headers)
148    print(len(renewed), "renewed")
149
150if __name__ == "__main__":
151    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
152    parser.add_argument("file", type=str, help="""file""")
153    parser.add_argument("-l", "--last_list", type=str, default=None, help="""last list""")
154    parser.add_argument("-o", "--out_dir", type=str, default="output", help="""output directory""")
155
156    args = parser.parse_args()
157    last_results = []
158    if args.last_list:
159        last_results, _ = utils.load_csv(args.last_list)
160        last_results = utils.clean_record(last_results)
161    list1, headers = utils.load_csv(args.file)
162    list1 = utils.clean_record(list1)
163
164    main(list1, headers, last_results, args.out_dir)
165
166
167##############
168### UTILS ####
169##############
170
171import csv
172
173def load_csv(filename):
174    with open(filename, 'r') as f:
175        headers = [s.lower() for s in f.read().split('\n')[0].split(',')]
176    with open(filename, 'r') as f:
177        reader = csv.DictReader(f, headers)
178        vals = list(reader)[1:]
179        return vals, headers
180
181def write_csv(filename, vals, headers=[]):
182    if not vals:
183        with open(filename, "w") as f:
184            f.write("")
185        return
186    if not headers:
187        headers = list(vals[0].keys())
188    with open(filename, 'w') as f:
189        header_vals = ",".join(headers)
190        f.write(header_vals + "\n")
191        vals = '\n'.join([','.join([r.get(k, "") or "" for k in headers]) for r in vals])
192        f.write(vals)
193
194#PITA, but needed for gmail
195def dedupe_emails(email):
196    if not email:
197        return None
198    email = email.lower().strip(", .")
199    splitted = email.split('@')
200    if splitted[1] == 'gmail.com':
201        usn = splitted[0].replace('.','')
202        email = "{}@gmail.com".format(usn)
203    return email
204
205def pad_year(year):
206    if year < 30:
207        year += 2000
208    elif year < 100:
209        year += 1900
210    return year
211
212def parse_date(date_str):
213    #Make sure it's always year-month-day
214    str_splitted = date_str.replace('/','-').split('-')
215    if len(str_splitted[2]) > 4:
216        str_splitted[2] = str_splitted[2].split(" ")[0]
217
218    date_splitted = [int(f) for f in str_splitted]
219    if date_splitted[0] > 12 and date_splitted[1] <= 12:
220        date_splitted[0] = pad_year(date_splitted[0])
221        return date_splitted
222    elif date_splitted[1] <= 12 and date_splitted[2] > 12:
223        return [pad_year(date_splitted[2]), date_splitted[0], date_splitted[1]]
224    elif date_splitted[1] > 12: #must be month-day-year
225        return [pad_year(date_splitted[2]), date_splitted[0], date_splitted[1]]
226    elif str_splitted[2][0] == '0': #must be month-day-year
227        return [pad_year(date_splitted[2]), date_splitted[0], date_splitted[1]]
228    elif len(str_splitted[0]) == 1: #must be month-day-year
229        return [pad_year(date_splitted[2]), date_splitted[0], date_splitted[1]]
230    else: #just assume it's month-day-year, it's impossible to tell
231        return [pad_year(date_splitted[2]), date_splitted[0], date_splitted[1]]
232
233
234def is_before(date1, date2):
235    if int(date1[0]) != int(date2[0]):
236        return int(date1[0]) < int(date2[0])
237    if int(date1[1]) != int(date2[1]):
238        return int(date1[1]) < int(date2[1])
239    if int(date1[2]) != int(date2[2]):
240        return int(date1[2]) < int(date2[2])
241
242def clean_record(records):
243    for row in records:
244        for val in ['xdate', 'join_date', 'x_date']:
245            if val in row:
246                row["{}_parsed".format(val)]  = parse_date(row[val])
247        for val in ['email', 'email1', 'email2', 'email3', 'email4']:
248            if val in row:
249                row[val]  = dedupe_emails(row[val])
250        for k, v in row.items():
251            if isinstance(v, str) and v.lower() == 'true':
252                row[k] = True
253            elif isinstance(v, str) and v.lower() == 'false':
254                row[k] = False
255            if isinstance(v, str):
256                row[k] = v.replace(",","").replace("\n","")
257    return records