· 5 years ago · Jun 18, 2020, 02:40 PM
1package frontend
2
3import (
4 "encoding/json"
5 "fmt"
6 "net/http"
7 "net/url"
8 "strconv"
9 "strings"
10 "time"
11
12 "github.com/teamyozax/yozax/bangs"
13 "github.com/teamyozax/yozax/instant"
14 "github.com/teamyozax/yozax/instant/discography"
15 "github.com/teamyozax/yozax/instant/parcel"
16 "github.com/teamyozax/yozax/instant/stock"
17 "github.com/teamyozax/yozax/instant/weather"
18 "github.com/teamyozax/yozax/instant/wikipedia"
19 "github.com/teamyozax/yozax/log"
20 "github.com/teamyozax/yozax/search"
21 img "github.com/teamyozax/yozax/search/image"
22 "github.com/pkg/errors"
23 "golang.org/x/text/language"
24)
25
26// Context holds a user's request context so we can pass it to our template's form.
27// Query, Language, and Region are the RAW query string variables.
28type Context struct {
29 Q string `json:"query"`
30 L string `json:"-"`
31 R string `json:"-"`
32 N string `json:"-"`
33 T string `json:"-"`
34 Safe bool `json:"-"`
35 DefaultBangs []DefaultBang `json:"-"`
36 Preferred []language.Tag `json:"-"`
37 Region language.Region `json:"-"`
38 Number int `json:"-"`
39 Page int `json:"-"`
40}
41
42// DefaultBang is the user's preffered @bang
43type DefaultBang struct {
44 Trigger string
45 bangs.Bang
46}
47
48// Results is the results from search, instant, wikipedia, etc
49type Results struct {
50 Alternative string `json:"alternative"`
51 Images *img.Results `json:"images"`
52 Instant instant.Data `json:"instant"`
53 Search *search.Results `json:"search"`
54}
55
56// Instant is a wrapper to facilitate custom unmarshalling
57type Instant struct {
58 instant.Data
59}
60
61type data struct {
62 Brand
63 Context `json:"-"`
64 Results
65}
66
67func (f *Frontend) defaultBangs(r *http.Request) []DefaultBang {
68 var bngs []DefaultBang
69
70 for _, db := range strings.Split(strings.TrimSpace(r.FormValue("b")), ",") {
71 for _, b := range f.Bangs.Bangs {
72 for _, t := range b.Triggers {
73 if t == db {
74 bngs = append(bngs, DefaultBang{db, b})
75 }
76 }
77 }
78 }
79
80 if len(bngs) > 0 {
81 return bngs
82 }
83
84 // defaults if no valid params passed
85 for _, b := range []struct {
86 trigger string
87 name string
88 }{
89 {"g", "Google"},
90 {"b", "Bing"},
91 {"a", "Amazon"},
92 {"yt", "Youtube"},
93 } {
94 for _, bng := range f.Bangs.Bangs {
95 if bng.Name == b.name {
96 bngs = append(bngs, DefaultBang{b.trigger, bng})
97 }
98 }
99 }
100
101 return bngs
102}
103
104// Detect the user's preferred language(s).
105// The "l" param takes precedence over the "Accept-Language" header.
106func (f *Frontend) detectLanguage(r *http.Request) []language.Tag {
107 preferred := []language.Tag{}
108 if lang := strings.TrimSpace(r.FormValue("l")); lang != "" {
109 if l, err := language.Parse(lang); err == nil {
110 preferred = append(preferred, l)
111 }
112 }
113
114 tags, _, err := language.ParseAcceptLanguage(r.Header.Get("Accept-Language"))
115 if err != nil {
116 log.Info.Println(err)
117 return preferred
118 }
119
120 preferred = append(preferred, tags...)
121 return preferred
122}
123
124// Detect the user's region. "r" param takes precedence over the language's region (if any).
125func (f *Frontend) detectRegion(lang language.Tag, r *http.Request) language.Region {
126 reg, err := language.ParseRegion(strings.TrimSpace(r.FormValue("r")))
127 if err != nil {
128 reg, _ = lang.Region()
129 }
130
131 return reg.Canonicalize()
132}
133
134func (f *Frontend) addQuery(q string) error {
135 exists, err := f.Suggest.Exists(q)
136 if err != nil {
137 return err
138 }
139
140 if !exists {
141 if err := f.Suggest.Insert(q); err != nil {
142 return err
143 }
144 }
145
146 return f.Suggest.Increment(q)
147}
148
149func (f *Frontend) searchHandler(w http.ResponseWriter, r *http.Request) *response {
150 q := strings.TrimSpace(r.FormValue("q"))
151 var safe = true
152 if strings.TrimSpace(r.FormValue("safe")) == "f" {
153 safe = false
154 }
155
156 resp := &response{
157 status: http.StatusOK,
158 data: data{
159 Brand: f.Brand,
160 Context: Context{
161 Safe: safe,
162 },
163 },
164 template: "search",
165 err: nil,
166 }
167
168 // render start page if no query
169 if q == "" {
170 return resp
171 }
172
173 d := data{
174 f.Brand,
175 Context{
176 Q: q,
177 L: strings.TrimSpace(r.FormValue("l")),
178 N: strings.TrimSpace(r.FormValue("n")),
179 R: strings.TrimSpace(r.FormValue("r")),
180 T: strings.TrimSpace(r.FormValue("t")),
181 Safe: safe,
182 DefaultBangs: f.defaultBangs(r),
183 },
184 Results{
185 Search: &search.Results{},
186 },
187 }
188
189 d.Context.Preferred = f.detectLanguage(r)
190 lang, _, _ := f.Document.Matcher.Match(d.Context.Preferred...) // will use first supported tag in case of error
191
192 d.Context.Region = f.detectRegion(lang, r)
193
194 // is it a @bang? Redirect them
195 if loc, ok := f.Bangs.Detect(d.Context.Q, d.Context.Region, lang); ok {
196 return &response{
197 status: 302,
198 redirect: loc,
199 }
200 }
201
202 // Let's get them their results
203 // what page are they on? Give them first page by default
204 var err error
205 d.Context.Page, err = strconv.Atoi(strings.TrimSpace(r.FormValue("p")))
206 if err != nil || d.Context.Page < 1 {
207 d.Context.Page = 1
208 }
209
210 // how many results wanted?
211 d.Context.Number, err = strconv.Atoi(strings.TrimSpace(r.FormValue("n")))
212 if err != nil || d.Context.Number > 100 {
213 d.Context.Number = 25
214 }
215
216 channels := 1
217 imageCH := make(chan *img.Results)
218 sc := make(chan *search.Results)
219 var ac chan error
220 var ic chan instant.Data
221
222 strt := time.Now() // we already have total response time in nginx...we want the breakdown
223
224 if d.Context.Page == 1 {
225 channels++
226
227 ac = make(chan error)
228 go func(q string, ch chan error) {
229 ch <- f.addQuery(q)
230 }(d.Context.Q, ac)
231
232 if d.Context.T != "images" {
233 channels++
234 ic = make(chan instant.Data)
235 go func(r *http.Request) {
236 lang, _, _ := f.Wikipedia.Matcher.Match(d.Context.Preferred...)
237 key := cacheKey("instant", lang, f.detectRegion(lang, r), r.URL)
238
239 v, err := f.Cache.Get(key)
240 if err != nil {
241 log.Info.Println(err)
242 }
243
244 if v != nil {
245 ir := &Instant{
246 instant.Data{},
247 }
248
249 if err := json.Unmarshal(v.([]byte), &ir); err != nil {
250 log.Info.Println(err)
251 }
252
253 ic <- ir.Data
254 return
255 }
256
257 res := f.Instant.Detect(r, lang)
258
259 if res.Cache {
260 var d = f.Cache.Instant
261
262 switch res.Type {
263 case "fedex", "ups", "usps", "stock quote", "weather": // only weather with a zip code gets cached "weather 90210"
264 d = 1 * time.Minute
265 }
266
267 if d > f.Cache.Instant {
268 d = f.Cache.Instant
269 }
270
271 if err := f.Cache.Put(key, res, d); err != nil {
272 log.Info.Println(err)
273 }
274 }
275
276 ic <- res
277 }(r)
278 }
279 }
280
281 go func(d data, lang language.Tag, region language.Region) {
282 switch d.Context.T {
283 case "images":
284 key := cacheKey("images", lang, region, r.URL)
285
286 v, err := f.Cache.Get(key)
287 if err != nil {
288 log.Info.Println(err)
289 }
290
291 if v != nil {
292 sr := &img.Results{}
293 if err := json.Unmarshal(v.([]byte), &sr); err != nil {
294 log.Info.Println(err)
295 }
296
297 imageCH <- sr
298 return
299 }
300
301 num := 100
302 offset := d.Context.Page*num - num
303 sr, err := f.Images.Fetch(d.Context.Q, d.Context.Safe, num, offset) // .8 is Yahoo's open_nsfw cutoff for nsfw
304 if err != nil {
305 log.Info.Println(err)
306 }
307
308 if err := f.Cache.Put(key, sr, f.Cache.Search); err != nil {
309 log.Info.Println(err)
310 }
311
312 imageCH <- sr
313 default:
314 key := cacheKey("search", lang, region, r.URL)
315
316 v, err := f.Cache.Get(key)
317 if err != nil {
318 log.Info.Println(err)
319 }
320
321 if v != nil {
322 sr := &search.Results{}
323 if err := json.Unmarshal(v.([]byte), &sr); err != nil {
324 log.Info.Println(err)
325 }
326
327 sc <- sr
328 return
329 }
330
331 // get the votes
332 offset := d.Context.Page*d.Context.Number - d.Context.Number
333 votes, err := f.Vote.Get(d.Context.Q, d.Context.Number*10) // get votes for first 10 pages
334 if err != nil {
335 log.Info.Println(err)
336 }
337
338 sr, err := f.Search.Fetch(d.Context.Q, lang, region, d.Context.Number, offset, votes)
339 if err != nil {
340 log.Info.Println(err)
341 }
342
343 for _, doc := range sr.Documents {
344 for _, v := range votes {
345 if doc.ID == v.URL {
346 doc.Votes = v.Votes
347 }
348 }
349 }
350
351 sr = sr.AddPagination(d.Context.Number, d.Context.Page) // move this to javascript??? (Wouldn't be available in API....)
352
353 if err := f.Cache.Put(key, sr, f.Cache.Search); err != nil {
354 log.Info.Println(err)
355 }
356
357 sc <- sr
358 }
359
360 }(d, lang, d.Context.Region)
361
362 stats := struct {
363 autocomplete time.Duration
364 images time.Duration
365 instant time.Duration
366 search time.Duration
367 }{}
368
369 for i := 0; i < channels; i++ {
370 select {
371 case d.Images = <-imageCH:
372 stats.images = time.Since(strt).Round(time.Millisecond)
373 case d.Instant = <-ic:
374 if d.Instant.Err != nil {
375 log.Info.Println(d.Instant.Err)
376 }
377 stats.instant = time.Since(strt).Round(time.Microsecond)
378 case d.Search = <-sc:
379 stats.search = time.Since(strt).Round(time.Millisecond)
380 case err := <-ac:
381 if err != nil {
382 log.Info.Println(err)
383 }
384 stats.autocomplete = time.Since(strt).Round(time.Millisecond)
385 case <-r.Context().Done():
386 // TODO: add info on which items took too long...
387 // Perhaps change status code of response so it isn't cached by nginx
388 log.Info.Println(errors.Wrapf(r.Context().Err(), "timeout on retrieving results"))
389 }
390 }
391
392 log.Info.Printf("ac:%v, images: %v, instant (%v):%v, search:%v\n", stats.autocomplete, stats.images, d.Instant.Type, stats.instant, stats.search)
393
394 if r.FormValue("o") == "json" {
395 resp.template = r.FormValue("o")
396 }
397
398 resp.data = d
399 return resp
400}
401
402func cacheKey(item string, lang language.Tag, region language.Region, u *url.URL) string {
403 // language and region might be different than what is pass as l & r params
404 // ::search::en-US::US::/?q=reverse+%22this%22
405 // ::instant::en-US::US::/?q=reverse+%22this%22
406 return fmt.Sprintf("::%v::%v::%v::%v", item, lang.String(), region.String(), u.String())
407}
408
409// UnmarshalJSON unmarshals an instant answer to the correct data structure
410func (d *Instant) UnmarshalJSON(b []byte) error {
411 type alias Instant
412 raw := &alias{}
413
414 err := json.Unmarshal(b, &raw)
415 if err != nil {
416 return err
417 }
418
419 j, err := json.Marshal(raw.Solution)
420 if err != nil {
421 return err
422 }
423
424 d.Data = raw.Data
425 d.Solution = raw.Solution
426
427 s := detectType(raw.Type)
428 if s == nil { // a string
429 return nil
430 }
431
432 d.Solution = s
433 return json.Unmarshal(j, d.Solution)
434}
435
436// detectType returns the proper data structure for an instant answer type
437func detectType(t string) interface{} {
438 var v interface{}
439
440 switch t {
441 case "discography":
442 v = &[]discography.Album{}
443 case "fedex", "ups", "usps":
444 v = &parcel.Response{}
445 case "stackoverflow":
446 v = &instant.StackOverflowAnswer{}
447 case "stock quote":
448 v = &stock.Quote{}
449 case "weather":
450 v = &weather.Weather{}
451 case "wikipedia":
452 v = &wikipedia.Item{}
453 case "wikidata age":
454 v = &instant.Age{
455 Birthday: &instant.Birthday{},
456 Death: &instant.Death{},
457 }
458 case "wikidata birthday":
459 v = &instant.Birthday{}
460 case "wikidata death":
461 v = &instant.Death{}
462 case "wikidata height", "wikidata weight":
463 v = &[]wikipedia.Quantity{}
464 case "wikiquote":
465 v = &[]string{}
466 case "wiktionary":
467 v = &wikipedia.Wiktionary{}
468 default: // a string
469 return nil
470 }
471
472 return v
473}