kDqRW9YR

1import collections
2import functools
3import itertools
4import json
5import math
6import urllib.parse
7
8import pandas
9import scrapy
10REPORTS = ["fflogs report IDs goes here", "e.g.,", "YA4QvwPCft1qdxBX"]
11API_KEY = "fflogs API key goes here"
12
13PHASES = ["Twin", "Nael", "Quickmarch", "Blackfire", "Fellruin", "Heavensfall", "Tenstrike", "Octet", "Adds", "Golden"] + \
14         ["Garuda (not woken)", "Garuda (woken)", "Ifrit (not woken)", "Ifrit (woken)", "Titan (not woken)", "Titan (woken)", "LB Phase", "Predation", "Annihilation", "Suppression", "Final"] + \
15         ["Pepsiman", "Limit Cut", "CC + BJ", "Time Stop + Inception", "Wormhole", "Perfect Alex"]
16TRIOS = {
17    9954: "Quickmarch",
18    9955: "Blackfire",
19    9956: "Fellruin",
20    9957: "Heavensfall",
21    9958: "Tenstrike",
22    9959: "Octet",
23}
24ULTIMATES = {
25    11126: "Predation",
26    11596: "Annihilation",
27    11597: "Suppression",
28    11151: "Final", # 3x Viscous Aetheroplasm
29}
30WOKEN_DEBUFF = 1000000 + 1529;
31
32TRIO_FILTER = "type = 'cast' AND (" + " OR ".join(map(lambda t: "ability.id = " + str(t), TRIOS.keys())) + ")"
33ULTIMATE_FILTER = "(type = 'cast' AND (" + " OR ".join(map(lambda t: "ability.id = " + str(t), ULTIMATES.keys())) + "))" + \
34    " OR (type = 'applydebuff' AND ability.id = " + str(WOKEN_DEBUFF) + ")"
35TITAN_GAOL_FILTER = "(type = 'death' AND target.disposition = 'friendly') OR ability.id = 11115 OR ability.id = 11116 OR (type = 'applydebuff' AND ability.id = 1000292)"
36
37class FFLogsSpider(scrapy.Spider):
38    name = "FFLogs"
39    custom_settings = {
40        "COOKIES_ENABLED": False,
41    }
42
43    fightData = {}
44    uwuRoleDps = {}
45    ucobRoleDps = {}
46
47    titanGaolData = {
48        "deaths": collections.Counter(),
49        "success": collections.Counter(),
50        "failPositioning": collections.Counter(),
51        "failDeathCount": 0,
52        "failPositioningCount": 0,
53    }
54
55    def start_requests(self):
56        for report in REPORTS:
57            yield scrapy.Request(
58                "https://www.fflogs.com/v1/report/fights/{}?api_key={}&translate=true".format(report, API_KEY),
59                functools.partial(self.fights, report)
60            )
61
62    def fights(self, reportId, response):
63        data = json.loads(response.body_as_unicode())
64
65        data["actors"] = {}
66        for actor in itertools.chain(data["friendlies"], data["enemies"]):
67            data["actors"][actor["id"]] = actor
68
69        uwuStart = math.inf
70        uwuEnd = -math.inf
71        uwuFights = []
72
73        teaStart = math.inf
74        teaEnd = -math.inf
75        teaFights = []
76
77        for fight in data["fights"]:
78            fight["report"] = data
79            fight["reportId"] = reportId
80            if fight["zoneName"] == "The Unending Coil Of Bahamut (Ultimate)":
81                if "lastPhaseForPercentageDisplay" not in fight or fight["lastPhaseForPercentageDisplay"] == 1:
82                    self.fightData.setdefault(reportId, []).append((fight, "Twin"))
83                elif fight["lastPhaseForPercentageDisplay"] == 2:
84                    self.fightData.setdefault(reportId, []).append((fight, "Nael"))
85                elif fight["lastPhaseForPercentageDisplay"] == 3:
86                    yield response.follow(
87                        "https://www.fflogs.com/v1/report/events/{}?".format(reportId) +
88                            urllib.parse.urlencode({
89                                "api_key": API_KEY,
90                                "start": fight["start_time"],
91                                "end": fight["end_time"],
92                                "filter": TRIO_FILTER
93                            }),
94                        functools.partial(self.ucobEvents, fight)
95                    )
96                elif fight["lastPhaseForPercentageDisplay"] == 4:
97                    self.fightData.setdefault(reportId, []).append((fight, "Adds"))
98                elif fight["lastPhaseForPercentageDisplay"] == 5:
99                    self.fightData.setdefault(reportId, []).append((fight, "Golden"))
100
101                    #if fight["bossPercentage"] <= 2000:
102                    if "kill" in fight and fight["kill"]:
103                        yield scrapy.Request(
104                            "https://www.fflogs.com/v1/report/tables/damage-done/{}?".format(fight["reportId"]) +
105                                urllib.parse.urlencode({
106                                    "api_key": API_KEY,
107                                    "start": fight["start_time"],
108                                    "end": fight["end_time"],
109                                    "filter": "IN RANGE FROM type = 'begincast' AND ability.id = 9964 TO type = 'cast' AND ability.id = 9965 END"
110                                }),
111                            functools.partial(self.ucobDamageDone, fight)
112                        )
113                else:
114                    assert(False)
115            elif fight["zoneName"] == "The Weapon's Refrain (Ultimate)":
116                if "lastPhaseForPercentageDisplay" not in fight:
117                    self.fightData.setdefault(reportId, []).append((fight, "Garuda (not woken)"))
118                else:
119                    uwuStart = min(uwuStart, fight["start_time"])
120                    uwuEnd = max(uwuEnd, fight["end_time"])
121                    uwuFights.append(fight)
122            elif fight["zoneName"] == "The Epic of Alexander (Ultimate)":
123                if "lastPhaseForPercentageDisplay" not in fight or fight["lastPhaseForPercentageDisplay"] == 1:
124                    if not fight.get("lastPhaseIsIntermission"):
125                        self.fightData.setdefault(reportId, []).append((fight, "Pepsiman"))
126                    else:
127                        self.fightData.setdefault(reportId, []).append((fight, "Limit Cut"))
128                elif (fight["lastPhaseForPercentageDisplay"] == 4 or
129                    (fight["lastPhaseForPercentageDisplay"] == 3 and fight.get("lastPhaseIsIntermission"))):
130                    self.fightData.setdefault(reportId, []).append((fight, "Perfect Alex"))
131                else:
132                    teaStart = min(teaStart, fight["start_time"])
133                    teaEnd = max(teaEnd, fight["end_time"])
134                    teaFights.append(fight)
135
136        if uwuStart < uwuEnd:
137            yield self.getAllEvents(
138                reportId, uwuStart, uwuEnd,
139                {"filter": "({}) OR ({})".format(ULTIMATE_FILTER, TITAN_GAOL_FILTER)},
140                functools.partial(self.uwuEvents, uwuFights)
141            )
142        if teaStart < teaEnd:
143            yield self.getAllEvents(
144                reportId, teaStart, teaEnd,
145                {"filter": "type = 'cast' AND (" +
146                    "ability.id = 18494 OR " + # Judgement Nisi
147                    "ability.id = 18522 OR " + # Temporal Stasis
148                    "ability.id = 18542" + # Wormhole
149                ")"},
150                functools.partial(self.teaEvents, teaFights)
151            )
152
153    def getAllEvents(self, reportId, start, end, args, callback):
154        args.update({
155            "api_key": API_KEY,
156            "start": start,
157            "end": end,
158        })
159        events = []
160
161        def continuation(response):
162            nonlocal args, events
163            data = json.loads(response.body_as_unicode())
164            events.extend(data["events"])
165            if len(data["events"]) != 0:
166                args["start"] = data["events"][-1]["timestamp"] + 1
167                return response.follow(
168                    "https://www.fflogs.com/v1/report/events/{}?".format(reportId) +
169                        urllib.parse.urlencode(args),
170                    continuation
171                )
172            else:
173                events = pandas.Series(events, index = map(lambda e: e["timestamp"], events))
174                return callback(events)
175
176        return scrapy.Request(
177            "https://www.fflogs.com/v1/report/events/{}?".format(reportId) +
178                urllib.parse.urlencode(args),
179            continuation
180        )
181
182    def ucobEvents(self, fight, response):
183        data = json.loads(response.body_as_unicode())
184        assert("nextPageTimestamp" not in data)
185
186        if len(data["events"]) == 0:
187            # Probably died to phase transition
188            self.fightData.setdefault(fight["reportId"], []).append((fight, "Nael"))
189        else:
190            self.fightData.setdefault(fight["reportId"], []).append((fight, TRIOS[data["events"][-1]["ability"]["guid"]]))
191
192    def ucobDamageDone(self, fight, response):
193        data = json.loads(response.body_as_unicode())
194
195        activeTime = max(e["activeTime"] for e in data["entries"]) / 1000
196        print("\nFight {}:{} Golden -> 1st Enrage hit DPS".format(fight["reportId"], fight["id"]))
197        for entry in data["entries"]:
198            if entry["type"] == "LimitBreak" and entry["name"] != "Limit Break":
199                continue
200            self.ucobRoleDps.setdefault(entry["type"], []).append(entry["total"] / activeTime)
201            print("{}: {:.0f} dps".format(entry["type"], entry["total"] / activeTime))
202        print("")
203
204    def uwuEvents(self, fights, allEvents):
205        def rfindAbility(events, abilities):
206            for e, event in events.iloc[::-1].items():
207                if "ability" not in event:
208                    continue
209                if event["ability"]["guid"] in abilities:
210                    return event
211            return None
212
213        for fight in fights:
214            events = allEvents.loc[fight["start_time"]:fight["end_time"]]
215
216            phase = None
217            lastWokenTarget = None
218            if fight["lastPhaseForPercentageDisplay"] == 4:
219                phase = "LB Phase"
220            elif fight["lastPhaseForPercentageDisplay"] == 5:
221                ultimateEvent = rfindAbility(events, ULTIMATES)
222                phase = ULTIMATES[ultimateEvent["ability"]["guid"]] if ultimateEvent else "LB Phase"
223
224                #if fight["bossPercentage"] <= 2000:
225                if "kill" in fight and fight["kill"]:
226                    yield scrapy.Request(
227                        "https://www.fflogs.com/v1/report/tables/damage-done/{}?".format(fight["reportId"]) +
228                            urllib.parse.urlencode({
229                                "api_key": API_KEY,
230                                "start": fight["start_time"],
231                                "end": fight["end_time"],
232                                "filter": "IN RANGE FROM encounterPhase = 5 AND type = 'begincast' AND ability.id = 11147 TO ability.id = 1000201 END"
233                            }),
234                        functools.partial(self.uwuDamageDone, fight)
235                    )
236            else:
237                lastWokenEvent = rfindAbility(events, {WOKEN_DEBUFF})
238                lastWokenTarget = fight["report"]["actors"][lastWokenEvent["targetID"]]["name"] if lastWokenEvent else None
239
240                if fight["lastPhaseForPercentageDisplay"] == 1:
241                    phase = "Garuda (woken)" if lastWokenTarget == "Garuda" else "Garuda (not woken)"
242                elif fight["lastPhaseForPercentageDisplay"] == 2:
243                    phase = "Ifrit (woken)" if lastWokenTarget == "Ifrit" else "Ifrit (not woken)"
244                elif fight["lastPhaseForPercentageDisplay"] == 3:
245                    phase = "Titan (woken)" if lastWokenTarget == "Titan" else "Titan (not woken)"
246
247            if fight["lastPhaseForPercentageDisplay"] >= 3:
248                isTitanWoken = lastWokenTarget == "Titan" or fight["lastPhaseForPercentageDisplay"] == 5
249                self.titanGaolEvents(fight, events.tolist(), isTitanWoken)
250
251            assert(phase != None)
252            self.fightData.setdefault(fight["reportId"], []).append((fight, phase))
253
254    def uwuDamageDone(self, fight, response):
255        data = json.loads(response.body_as_unicode())
256
257        activeTime = max(e["activeTime"] for e in data["entries"]) / 1000
258        print("\nFight {}:{} Ultima -> Stun DPS".format(fight["reportId"], fight["id"]))
259        for entry in data["entries"]:
260            if entry["type"] == "LimitBreak" and entry["name"] != "Limit Break":
261                continue
262            self.uwuRoleDps.setdefault(entry["type"], []).append(entry["total"] / activeTime)
263            print("{}: {:.0f} dps".format(entry["type"], entry["total"] / activeTime))
264        print("")
265
266    def titanGaolEvents(self, fight, allEvents, isWoken):
267        start = -1
268        end = -1
269        gaols = 0
270        for e, event in enumerate(allEvents):
271            if "ability" not in event:
272                continue
273            if start == -1 and event["ability"]["guid"] in {11115, 11116}:
274                start = e
275            if gaols < 3 and event["ability"]["guid"] == 1000292:
276                gaols += 1
277                end = e + 1
278        if start == -1 or end < start + 6:
279            return
280        events = allEvents[start:end]
281
282        targeted = set()
283        for event in events[:3]:
284            if "ability" not in event or event["ability"]["guid"] not in {11115, 11116}:
285                return
286            targeted.add(event["targetID"])
287
288        deaths = set()
289        for event in events[3:-3]:
290            assert(event["type"] == "death")
291            deaths.add(event["targetID"])
292        for event in reversed(events[:start]):
293            if event["timestamp"] + 5000 < allEvents[start]["timestamp"]:
294                break
295            if event["type"] == "death":
296                deaths.add(event["targetID"])
297
298        gaoled = set()
299        for event in events[-3:]:
300            if "ability" not in event or event["ability"]["guid"] != 1000292:
301                return
302            gaoled.add(event["targetID"])
303
304        self.titanGaolData["deaths"].update(map(lambda id: fight["report"]["actors"][id]["name"], deaths))
305        if targeted == gaoled:
306            if isWoken:
307                self.titanGaolData["success"].update(map(lambda id: fight["report"]["actors"][id]["name"], targeted))
308            else:
309                self.titanGaolData["failPositioning"].update(map(lambda id: fight["report"]["actors"][id]["name"], targeted))
310                self.titanGaolData["failPositioningCount"] += 1
311        else:
312            self.titanGaolData["failDeathCount"] += 1
313
314    def teaEvents(self, fights, allEvents):
315        for fight in fights:
316            events = allEvents.loc[fight["start_time"]:fight["end_time"]]
317
318            if len(events) == 0:
319                # Before Nisi
320                self.fightData.setdefault(fight["reportId"], []).append((fight, "Limit Cut"))
321            elif events.iloc[-1]["ability"]["guid"] == 18494: # Judgement Nisi
322                self.fightData.setdefault(fight["reportId"], []).append((fight, "CC + BJ"))
323            elif events.iloc[-1]["ability"]["guid"] == 18522: # Temporal Stasis
324                self.fightData.setdefault(fight["reportId"], []).append((fight, "Time Stop + Inception"))
325            elif events.iloc[-1]["ability"]["guid"] == 18542: # Wormhole
326                self.fightData.setdefault(fight["reportId"], []).append((fight, "Wormhole"))
327            else:
328                assert(False)
329
330    def closed(self, reason):
331        MSPERHOUR = 1000 * 60 * 60
332
333        totalDuration = 0
334        phaseCounts = dict.fromkeys(PHASES, 0)
335        phaseDurations = dict.fromkeys(PHASES, 0)
336        phaseIntervals = {}
337        weekStarts = [0]
338        lastWeek = None
339        clears = []
340        for report in REPORTS:
341            for fight, phase in sorted(self.fightData[report], key = lambda e: e[0]["start_time"]):
342                duration = fight["end_time"] - fight["start_time"]
343                phaseCounts[phase] += 1
344                phaseDurations[phase] += duration
345                phaseIntervals.setdefault(phase, []).append((totalDuration, totalDuration + duration))
346
347                if lastWeek is None:
348                    lastWeek = fight["report"]["start"] + fight["start_time"]
349                if lastWeek + MSPERHOUR * 24 * 7 < fight["report"]["start"] + fight["end_time"]:
350                    weekStarts.append(totalDuration)
351                    while lastWeek + MSPERHOUR * 24 * 7 < fight["report"]["start"] + fight["end_time"]:
352                        lastWeek += MSPERHOUR * 24 * 7
353
354                isClear = "kill" in fight and fight["kill"]
355                if isClear:
356                    clears.append(totalDuration + duration)
357
358                totalDuration += duration
359
360                print("{}:{}: {}{}".format(fight["reportId"], fight["id"], phase, " (Clear)" if isClear else ""))
361        print("")
362        print("legends = {{{}}};".format(
363            ",".join(map(
364                lambda p: "\"{}\"".format(p),
365                filter(lambda p: p in phaseIntervals, PHASES)
366            ))
367        ))
368        print("data = {{{}}};".format(
369            ",".join(map(
370                lambda p: "{{{}}}".format(
371                    ",".join(map(lambda i: "{{{:.2f}, {:.2f}}}".format(i[0] / MSPERHOUR, i[1] / MSPERHOUR), phaseIntervals[p]))
372                ),
373                filter(lambda p: p in phaseIntervals, PHASES)
374            ))
375        ))
376        print("gridLines = {{{{{}}}, None}};".format(
377            ",".join(itertools.chain(
378                map(lambda c: "{{{:.2f}, Directive[Red, Thick]}}".format(c / MSPERHOUR), clears),
379                map(lambda w: "{:.2f}".format(w / MSPERHOUR), weekStarts)
380            ))
381        ))
382        print("""
383NumberLinePlot[
384    Interval @@ # & /@ data, PlotLegends -> legends,
385    PlotStyle -> Directive[Thickness[0.01], CapForm[None]],
386    AspectRatio -> 1 / 5, ImageSize -> 1000
387] /. Point[a_] -> Point[{-100, 1}]""")
388        print("""
389filling = Join[{{1 -> Axis}}, Table[{n -> {n - 1}}, {n, 2, Length[data]}]];
390windowSize = 4;
391smooth[fn_] := MovingAverage[Table[If[fn, 1, 0], {x, 0 - windowSize/2, Max[data] + windowSize/2, 0.01}], 100 windowSize];
392envelope = smooth[0 <= x < Max[data]];
393ParallelMap[smooth[Or @@ (#[[1]] <= x < #[[2]] & /@ #)] / envelope &, data];
394ListLinePlot[
395    Accumulate[%], Filling -> filling, DataRange -> {0, Max[data]},
396    PlotRange -> All, PlotLegends -> legends,
397    GridLines -> gridLines, GridLinesStyle -> Dashed,
398    PlotLabel -> "Time spent in pull wiping to phase x",
399    AxesLabel -> {"Hours in pull", "Ratio"},
400    ImageSize -> 1000
401]""")
402
403        print("")
404        for phase in PHASES:
405            print("{}: {} pulls, {:.1f}% in duration".format(phase, phaseCounts[phase], phaseDurations[phase] / totalDuration * 100))
406
407        if len(self.ucobRoleDps) > 0:
408            print("")
409            print("{{{}}}".format(",".join("plot[{0}, \"{0}\", {{{1}}}]".format(role, ",".join(map(str, dps))) for role, dps in self.ucobRoleDps.items())))
410
411        if len(self.uwuRoleDps) > 0:
412            print("")
413            print("{{{}}}".format(",".join("plot[{0}, \"{0}\", {{{1}}}]".format(role, ",".join(map(str, dps))) for role, dps in self.uwuRoleDps.items())))
414
415        if len(self.titanGaolData["deaths"]) > 0:
416            print("\nTitan Knockback Deaths")
417            print("\n".join(map(lambda e: "{}: {}".format(e[0], e[1]), self.titanGaolData["deaths"].most_common())))
418        if len(self.titanGaolData["failPositioning"]) > 0:
419            failureRate = collections.Counter()
420            for id, failCount in self.titanGaolData["failPositioning"].items():
421                successCount = self.titanGaolData["success"][id]
422                failureRate[id] = failCount / (failCount + successCount)
423
424            print("\nTitan Gaol Positioning Failure Rate")
425            print("\n".join(map(lambda e: "{}: {:.3f}".format(e[0], e[1]), failureRate.most_common())))
426        totalFailCount = self.titanGaolData["failDeathCount"] + self.titanGaolData["failPositioningCount"]
427        if totalFailCount > 0:
428            print("\nTitan Non-woken Reason:")
429            print("Knockback Death: {:.3f}".format(self.titanGaolData["failDeathCount"] / totalFailCount))
430            print("Gaol Positioning: {:.3f}".format(self.titanGaolData["failPositioningCount"] / totalFailCount))