· 9 years ago · Dec 12, 2016, 11:51 PM
1'use strict';
2
3const ig = require('./lib/indiegala');
4const steam = require('./lib/steam');
5const priority = require('./lib/enter-giveaways');
6
7steam.updateOwnedGames();
8
9ig.parseGiveawaysList()
10 .then(steam.scrape);
11
12priority
13 .prioritizeGiveaways()
14 .then( ig.enterGiveaways )
15 .then( ig.checkWins )
16 .then( ig.close )
17 .catch( (err) => {
18 console.error(err);
19 process.exit();
20 });
21
22'use strict';
23
24// modules
25const nconf = require('./config');
26const Nightmare = require('nightmare');
27const async = require('async');
28const request = require('request');
29const cheerio = require('cheerio');
30const model = require('./model');
31const fs = require('fs');
32const Q = require('q');
33const colors = require('colors');
34
35// config / globals
36const baseUrl = nconf.get('indiegala:baseUrl');
37
38// IndieGala Constants
39const IG_NOT_AUTHORIZED = 'You are not authorized access for this giveaway.';
40const IG_NOT_ENOUGH_COINS = 'Insufficient Indiegala Coins. Please choose a cheaper giveaway.';
41
42/**
43 * The number of giveaway listing pages to parse. There are 12 giveaways per page.
44 *
45 * @type {number}
46 */
47const pagesToParse = nconf.get('indiegala:pagesToParse');
48
49
50// setup Nightmare instance
51// require('nightmare-iframe-manager')(Nightmare);
52const nmConfig = nconf.get('nightmare');
53nmConfig.show = true; // show for manual login
54const nmInst = Nightmare(nmConfig);
55nmInst.useragent(nmConfig.userAgent);
56
57
58/**
59 * Process giveaway pages using Request/Cheerio to grab data and store in DB.
60 * @param {string[]} links Relative links to individual giveaways
61 */
62function processGiveawayPages(links) {
63 async.each(links, (link, next) => {
64 const giveawayUrl = baseUrl + link;
65 request(giveawayUrl, (err, resp, body) => {
66 if (err) {
67 console.error('Error loading giveaway url: n' + err);
68 next(err);
69 return;
70 }
71
72 const $ = cheerio.load(body);
73
74 // get end date from thier JS and convert to unix ts
75 let endTime = -1;
76 const found = body.match(/new Date(Date.UTC.*;/g);
77 if (null !== found) {
78 let getEndTime = new Function('return ' + found[0]);
79 endTime = Math.round(getEndTime().getTime() / 1000);
80 }
81
82 // get the steam url and remove trailing slashes
83 let steamUrl = $('.ticket-info-cont .steam-link').attr('href');
84 if (steamUrl.endsWith('/')) {
85 steamUrl = steamUrl.slice(0, -1);
86 }
87
88 const steamId = steamUrl.substring(steamUrl.lastIndexOf('/') + 1);
89
90 const giveaway = {
91 'id': $('.ticket-right .relative').attr('rel'),
92 'name': $('.ticket-info-cont h2').text(),
93 'price': $('.ticket-left .ticket-price strong').text(),
94 'endDate': endTime,
95 'level': $('.type-level-cont').text().trim(),
96 steamUrl,
97 steamId
98 };
99
100 model.insertGiveaway(giveaway);
101
102 next();
103 });
104 }, (err) => {
105 // all finished
106 if (err) {
107 console.error(err);
108 }
109 });
110}
111
112
113/////////////////////// Electron Context Functions ////////////////////
114
115/**
116 * Scrape the links to individual game links. This is run in Nightmare/Electron browser context.
117 *
118 * @return {string[]} Array of URLs to individual giveaway pages
119 */
120function parseGameLinks() {
121 // eslint-disable-next-line no-var, no-undef
122 var links = document.querySelectorAll('.ticket-info-cont h2 a');
123
124 return Array.prototype.map.call(links, function(e) {
125 return e.getAttribute('href');
126 });
127}
128
129/**
130 * Returns data about the attempted giveaway. This is run in Nightmare/Electron browser context.
131 *
132 * @return {Object}
133 */
134function getDataFromGiveaway() {
135 return {
136 entered: $('.giv-coupon').length === 0,
137 title: document.title,
138 coins: document.querySelector('.coins-amount').title,
139 error: $('.warning-cover:visible span').text()
140 };
141}
142
143////////////////////////////////////////////////////////////////////
144
145/**
146 * Parses pages of giveaways in series, count and level set in config.
147 *
148 * On each page, grab the URLs to each Giveaway details page. Then pass the
149 * urls to {@link processGiveawayPages} for individual parsing.
150 *
151 * @returns {Promise}
152 */
153function parseGiveawaysList() {
154 const deferred = Q.defer();
155 // Setup local Nightmare and start processing pages
156 const nmLocal = Nightmare(nconf.get('nightmare'));
157 nmLocal.useragent(nconf.get('nightmare:userAgent'));
158
159 async.timesSeries(pagesToParse + 1, (n, next) => {
160 // timesSeries is zero-based, the links are 1 based so skip
161 if (0 === n) {
162 next();
163 }
164 else {
165 const level = nconf.get('indiegala:level');
166 const thisPage = baseUrl + '/giveaways/' + n + '/expiry/asc/level/' + level;
167 nmLocal
168 .goto(thisPage)
169 .wait('.giveaways-main-page')
170 .evaluate(parseGameLinks)
171 .then((links) => {
172 // console.log(links);
173 processGiveawayPages(links);
174 console.log('Finished ' + thisPage);
175 next(null, links);
176 })
177 .catch((err) => {
178 console.error('Nightmare error loading giveaway list page: ');
179 console.error(colors.red(err));
180 next(err, links);
181 });
182 }
183
184 }, (err, allLinks) => {
185 if (err) {
186 console.error('All Finished Error: ' + err);
187 return;
188 }
189
190
191 setTimeout(() => {
192 close(nmLocal);
193 deferred.resolve();
194 }, 3000);
195 });
196
197 return deferred.promise;
198}
199
200/**
201 * Insert a random delay, between configurable milliseconds using nconf for defaults
202 *
203 * IG will log you out without this (1 to 3 seconds seems to be the min)
204 */
205function rndDelay(low, high) {
206 low = low || nconf.get('delayMs:low');
207 high = high || nconf.get('delayMs:high');
208
209 return Math.floor(Math.random() * (high - low + 1) + low);
210}
211
212
213//////////////////// Enter Giveaway functions ////////////////
214
215/**
216 * Next item callback for async.each style
217 *
218 * @see {@link http://caolan.github.io/async/docs.html#.each | async.each Docs}
219 * @callback asyncNextItem
220 * @param {(string|Object)} [error]
221 */
222
223/**
224 *
225 * @param {Object} giveaway - Giveaway obj with url and id
226 * @param {Object} data - Results obj returned from {@link getDataFromGiveaway}
227 * @param {asyncNextItem} next - Callback to continue to next item
228 * @return {bool}
229 */
230function shouldRetryGiveaway(giveaway, data, next) {
231 console.log(data.coins + 't' + data.title);
232
233 if (data.entered) {
234 model.markAsEntered(giveaway.id);
235 next();
236
237 return false;
238 }
239
240 if (data.coins === '0 Indiegala Coins') {
241 next('No More Coins');
242
243 return false;
244 }
245
246 if (data.error === IG_NOT_AUTHORIZED) {
247 console.log('--- Level Not High Enough');
248 next();
249
250 return false;
251 }
252
253 if (data.error === IG_NOT_ENOUGH_COINS) {
254 console.log('--- Not Enough Coins');
255 next();
256
257 return false;
258 }
259
260
261 return true;
262}
263
264
265
266/**
267 * For Nightmare.use() - goes to the url, clicks enter and
268 * runs {@link getDataFromGiveaway} in the Electron context.
269 */
270function gotoAndClickTicket(url) {
271 return (nightmare) => {
272 nightmare
273 .goto(url)
274 .wait(rndDelay())
275 .click('.giv-coupon')
276 .wait(rndDelay())
277 .evaluate(getDataFromGiveaway);
278 }
279}
280
281/**
282 * Enter the supplied giveaways and mark them as such
283 *
284 * @returns {Promise}
285 */
286function enterGiveaways(giveaways) {
287 console.log(colors.green('Giveaways In Queue: ' + giveaways.length));
288
289 const deferred = Q.defer();
290
291 async.eachSeries(giveaways, (giveaway, next) => {
292 nmInst
293 .use(gotoAndClickTicket(giveaway.url))
294 .then( (data) => {
295 // console.log(data);
296 if (shouldRetryGiveaway(giveaway, data, next)) {
297 console.log('Could not dectect succsessful entery, retry'.yellow);
298 return nmInst
299 .use(gotoAndClickTicket(giveaway.url))
300 .then( (data) => {
301 if (shouldRetryGiveaway(giveaway, data, next)) {
302 next();
303 }
304 });
305 }
306 })
307 .catch((err) => {
308 if (err === 'Unable to find element by selector: .giv-coupon') {
309 model.markAsEntered(giveaway.id);
310 next();
311 }
312 else {
313 console.error(err);
314 next('Nightmare Error entering giveaway: ' + err.message);
315 }
316 });
317 }, (err) => {
318 if (err) {
319 console.error(err);
320 // deferred.reject(err);
321 }
322 // all giveaways entered
323 console.log('All Giveaways Entered');
324 deferred.resolve();
325 });
326
327 return deferred.promise;
328}
329
330/**
331 * Clicks all the 'Check if you won!' buttons
332 */
333function clickAllCheckForWinButtons(buttonCount, deferred) {
334 console.log('Found Buttons: ' + buttonCount);
335
336 async.timesSeries(buttonCount, (n, next) => {
337 nmInst
338 .click('.btn-check-if-won')
339 // without this wait it tries to click the same button
340 .wait(rndDelay(4000, 6000))
341 .then(() => {
342 next();
343 });
344 }, err => {
345 if (err) {
346 console.error(err);
347 deferred.reject(err);
348 }
349 // all giveaways entered
350 console.log('All Completed Giveaways Checked');
351 deferred.resolve();
352 });
353}
354
355/**
356 * Go to profile page and check any completed ones for wins
357 */
358function checkWins() {
359 console.log('Check for Wins');
360 const deferred = Q.defer();
361
362 nmInst
363 .goto('https://www.indiegala.com/profile')
364 .wait('#open-giveaways-library')
365 .click('#open-giveaways-library')
366 .click('.giveaway-completed .open-library')
367 .wait(rndDelay(4000, 6000))
368 .evaluate(() => {
369 return document.querySelectorAll('.btn-check-if-won').length;
370 })
371 .then((buttonCount) => {
372 clickAllCheckForWinButtons(buttonCount, deferred);
373 })
374 .catch((err) => {
375 console.error(err);
376 });
377
378 return deferred.promise;
379}
380
381
382/**
383 * End the Nightmare/Electron session
384 */
385function close(aNightmareInst) {
386 aNightmareInst = aNightmareInst || nmInst;
387 aNightmareInst
388 .goto(baseUrl)
389 .end()
390 .catch((err) => {
391 console.error(err);
392 });
393}
394
395module.exports = {
396 login,
397 checkWins,
398 enterGiveaways,
399 parseGiveawaysList,
400 close
401};
402
403'use strict';
404const nconf = require('./config');
405const Q = require('q');
406const sqlite3 = require('sqlite3').verbose();
407const db = new sqlite3.Database(nconf.get('sqliteFile'));
408
409sqlite3.verbose();
410
411// setup tables
412const createGamesTableSql = `
413 CREATE TABLE IF NOT EXISTS games (
414 steamId INTEGER,
415 reviewText TEXT,
416 reviewStats TEXT,
417 genre TEXT,
418 metascore INTEGER,
419 tag1 TEXT,
420 tag2 TEXT,
421 tag3 TEXT,
422 shortDesc TEXT,
423 PRIMARY KEY(steamId)
424 );
425`;
426
427const creatGiveawaysTableSql = `
428 CREATE TABLE IF NOT EXISTS giveaways (
429 id INTEGER,
430 name TEXT,
431 steamUrl TEXT,
432 price NUMERIC,
433 endDate INTEGER,
434 steamId INTEGER,
435 entered INTEGER DEFAULT 0,
436 PRIMARY KEY(id)
437 );
438`;
439
440const createGamesOwnedTableSql = `
441 CREATE TABLE IF NOT EXISTS games_owned (
442 steamId INTEGER,
443 PRIMARY KEY(steamID)
444 );
445`;
446
447// setup queries
448
449
450const newGamesToDetailSql = `
451 SELECT DISTINCT steamUrl, steamId
452 FROM giveaways
453 WHERE steamId IS NOT NULL
454 AND steamId NOT IN (SELECT DISTINCT steamId FROM games)
455`;
456
457db.serialize();
458db.run(createGamesTableSql);
459db.run(creatGiveawaysTableSql);
460db.run(createGamesOwnedTableSql);
461db.parallelize();
462
463// giveaways queries
464const insertGiveaway = db.prepare(
465 'INSERT INTO giveaways (id, name, steamUrl, price, endDate, steamId) ' +
466 'VALUES (?, ?, ?, ?, ?, ?)');
467
468const markAsEntered = db.prepare('UPDATE giveaways SET entered = 1 WHERE id = ?');
469
470// games queries
471const insertGame = db.prepare('INSERT INTO games VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
472
473// games_owned queries
474const insertOwnedGame = db.prepare('INSERT INTO games_owned VALUES (?)');
475
476
477module.exports = {
478
479 // direct interface to the db
480 db,
481
482 // set giveaways.entered = 1 for the provided id
483 markAsEntered: (giveawayId) => {
484 markAsEntered.run(giveawayId, (updateError) => {
485 if (updateError) {
486 console.error(updateError);
487 }
488 else {
489 // console.log('Updated ' + giveawayId);
490 }
491 });
492 },
493
494 // insert new row to giveaways
495 insertGiveaway: (giveaway) => {
496 insertGiveaway.run(
497 giveaway.id,
498 giveaway.name,
499 giveaway.steamUrl,
500 giveaway.price,
501 giveaway.endDate,
502 giveaway.steamId,
503 (insertErr) => {
504 if (insertErr) {
505 // ignore PK insert errors
506 if (! insertErr.message.includes('UNIQUE constraint failed')) {
507 console.error(insertErr);
508 }
509 }
510 }
511 );
512 },
513
514 // insert new row to games
515 insertGame: (game) => {
516 insertGame.run(
517 game.steamId,
518 game.reviewText,
519 game.reviewStats,
520 game.genre,
521 game.metascore,
522 game.tag1,
523 game.tag2,
524 game.tag3,
525 game.shortDesc,
526 (insertErr) => {
527 if (insertErr) {
528 // ignore PK insert errors
529 if (! insertErr.message.includes('UNIQUE constraint failed')) {
530 console.error(insertErr);
531 }
532 }
533 }
534 );
535 },
536
537 // insert new row to games
538 insertOwnedGame: (steamId) => {
539 insertOwnedGame.run(
540 steamId,
541 (insertErr) => {
542 if (insertErr) {
543 // ignore PK insert errors
544 if (! insertErr.message.includes('UNIQUE constraint failed')) {
545 console.error(insertErr);
546 }
547 }
548 }
549 );
550 },
551
552 // returns promise for array of objs with steamUrl and steamId
553 getNewGamesToDetail: () => {
554 return new Q.Promise((fulfill, reject) => {
555 db.all(newGamesToDetailSql, (err, rows) => {
556 if (err) {
557 console.error(err);
558 reject(err);
559 }
560
561 fulfill(rows);
562 });
563 });
564 }
565};