· 4 years ago · Apr 13, 2021, 04:16 AM
1// This script populates the database with all the courses from Princeton's Registrar
2console.log('Starting script to update our database with latest course listings information from the Registrar.')
3
4// Load config variables from the .env file
5require('dotenv').config()
6
7// Load external dependencies
8const log = require('loglevel')
9const cheerio = require('cheerio')
10const request = require('request')
11const throttledRequest = require('throttled-request')(request)
12
13// Set the level of the logger to the first command line argument
14// Valid values: "trace", "debug", "info", "warn", "error"
15if (process.argv.length > 3) {
16 log.setLevel(process.argv[3])
17}
18
19// Load internal modules
20var semesterModel = require('../models/semester.js')
21var courseModel = require('../models/course.js')
22
23// Connect to the database
24require('../controllers/database.js')
25
26// This is the mapping from assignment keys to their proper titles
27const assignmentMapping = {
28 'grading_mid_exam': 'Mid term exam',
29 'grading_paper_mid_exam': 'Paper in lieu of mid term',
30 'grading_final_exam': 'Final exam',
31 'grading_paper_final_exam': 'Paper in lieu of final',
32 'grading_other_exam': 'Other exam',
33 'grading_home_mid_exam': 'Take home mid term exam',
34 'grading_design_projects': 'Design project',
35 'grading_home_final_exam': 'Take home final exam',
36 'grading_prog_assign': 'Programming assignments',
37 'grading_quizzes': 'Quizzes',
38 'grading_lab_reports': 'Lab reports',
39 'grading_papers': 'Papers',
40 'grading_oral_pres': 'Oral presentation(s)',
41 'grading_term_papers': 'Term paper(s)',
42 'grading_precept_part': 'Class/precept participation',
43 'grading_prob_sets': 'Problem set(s)',
44 'grading_other': 'Other (see instructor)'
45}
46const assignmentPropertyNames = Object.keys(assignmentMapping)
47
48// This is how we keep track of the token that is required for accessing api.princeton.edu
49let registrarFrontEndAPIToken
50
51// This will throttle web requests so no more than 5 are made every second
52throttledRequest.configure({
53 requests: 5,
54 milliseconds: 1000
55})
56
57// A function that takes a query string for the OIT's Course Offerings API and return to the
58// external callback junction a JSON object of the response data.
59// For example, the query "term=1174&subject=COS" will return all COS courses in
60// the Spring 2017 semester. Learn about valid query strings at https://webfeeds.princeton.edu/#feed,19
61var loadCoursesFromRegistrar = function (query, externalCallback) {
62 console.log("Preparing to make request to the Registrar for course listings data with query '%s'.", query)
63
64 request(`http://etcweb.princeton.edu/webfeeds/courseofferings/?fmt=json&vers=1.5&${query}`, function (error, response, body) {
65 if (error) {
66 return console.log(error)
67 }
68 externalCallback(JSON.parse(body))
69 })
70}
71
72var importDataFromRegistrar = function (data) {
73 console.log('Processing data recieved from the Registrar.')
74
75 for (var termIndex in data.term) {
76 var term = data.term[termIndex]
77 importTerm(term)
78 }
79}
80
81// Recieve a "term" of data (of the kind produced by the Registrar) and add/update the database to contain this data
82var importTerm = function (term) {
83 console.log('Processing the %s semester.', term.cal_name)
84
85 // Update/Add Semesters to the database
86 // Existing semesters not in data object will be untouched
87 // Existing semesters in data object will be updated
88 // New semesters in data object will be created
89 semesterModel.findOneAndUpdate({
90 _id: term.code
91 }, {
92 _id: term.code,
93 name: term.cal_name,
94 start_date: term.start_date,
95 end_date: term.end_date
96 }, {
97 new: true,
98 upsert: true,
99 runValidators: true,
100 setDefaultsOnInsert: true
101 }, function (error, semester) {
102 if (error) {
103 log.warn('Creating or updating the semester %s failed.', term.cal_name)
104 }
105 log.trace('Creating or updating the semester %s succeeded.', term.cal_name)
106
107 // Process each subject within this semester
108 for (var subjectIndex in term.subjects) {
109 var subject = term.subjects[subjectIndex]
110 importSubject(semester, subject)
111 }
112 })
113}
114
115// Decode escaped HTML characters in a string, for example changing "Foo&bar" to "Foo&bar"
116var decodeEscapedCharacters = function (html) {
117 return cheerio('<div>' + cheerio('<div>' + html + '</div>').text() + '</div>').text()
118}
119
120var importSubject = async function (semester, subject) {
121 log.debug('Processing the subject %s in the %s semester.', subject.code, semester.name)
122
123 // Iterate over the courses in this subject
124 for (var courseIndex in subject.courses) {
125 const courseData = subject.courses[courseIndex]
126
127 // Print the catalog number
128 //console.log('\t' + courseData.catalog_number)
129
130 if (typeof (courseData.catalog_number) === 'undefined' || courseData.catalog_number.length < 2) {
131 continue
132 }
133
134 // Decode escaped HTML characters in the course title
135 if (typeof (courseData.title) !== 'undefined') {
136 courseData.title = decodeEscapedCharacters(courseData.title)
137 }
138
139 // Decode escaped HTML characters in the course description
140 if (typeof (courseData.detail.description) !== 'undefined') {
141 courseData.detail.description = decodeEscapedCharacters(courseData.detail.description)
142 }
143
144 const requestOptions = {
145 url: `https://api.princeton.edu/registrar/course-offerings/course-details?term=${semester._id}&course_id=${courseData.course_id}`,
146 headers: {
147 Pragma: 'no-cache',
148 Accept: 'application/json',
149 Authorization: `Bearer ${registrarFrontEndAPIToken}`,
150 'User-Agent': 'Princeton Courses (https://www.princetoncourses.com)'
151 }
152 }
153
154 // Increment the number of courses pending processing
155 coursesPendingProcessing++
156
157 throttledRequest(requestOptions, function (error, response, body) {
158 if (error) {
159 coursesPendingProcessing--
160 return console.log(error)
161 }
162 console.log(`Got results for ${courseData.course_id}`)
163 let frontEndApiCourseDetails
164 try {
165 frontEndApiCourseDetails = JSON.parse(body).course_details.course_detail[0]
166 } catch (error) {
167 return console.error(error)
168 }
169
170 // Grading Basis
171 switch (frontEndApiCourseDetails.grading_basis) {
172 case 'FUL': // Graded A-F, P/D/F, Audit
173 courseData.pdf = {
174 required: false,
175 permitted: true
176 }
177 courseData.audit = true
178 break
179 case 'NAU': // No Audit
180 courseData.pdf = {
181 required: false,
182 permitted: true
183 }
184 courseData.audit = false
185 break
186 case 'GRD': // na, npdf
187 courseData.pdf = {
188 required: false,
189 permitted: false
190 }
191 courseData.audit = false
192 break
193 case 'NPD': // No Pass/D/Fail
194 courseData.pdf = {
195 required: false,
196 permitted: false
197 }
198 courseData.audit = true
199 break
200 case 'PDF': // P/D/F Only
201 courseData.pdf = {
202 required: true,
203 permitted: true
204 }
205 courseData.audit = true
206 break
207 default:
208 courseData.pdf = {
209 required: false,
210 permitted: true
211 }
212 courseData.audit = true
213 }
214
215 // Get Grading
216 courseData.grading = Object.keys(frontEndApiCourseDetails).filter(key => assignmentPropertyNames.includes(key)).filter(key => frontEndApiCourseDetails[key] !== '0').map(function (key) {
217 return {
218 component: assignmentMapping[key],
219 weight: parseFloat(frontEndApiCourseDetails[key])
220 }
221 }).sort(function (a, b) {
222 if (a.weight < b.weight) {
223 return 1
224 }
225 if (a.weight > b.weight) {
226 return -1
227 }
228 return 0
229 })
230
231 // Get assignments description
232 if (frontEndApiCourseDetails.reading_writing_assignment && frontEndApiCourseDetails.reading_writing_assignment.trim().length > 0) {
233 courseData.assignments = frontEndApiCourseDetails.reading_writing_assignment.trim()
234 }
235
236 // Get reserved seats
237 if (frontEndApiCourseDetails.seat_reservations.seat_reservation) {
238 courseData.reservedSeats = frontEndApiCourseDetails.seat_reservations.seat_reservation.map(reservation => `${reservation.description} ${reservation.enrl_cap}`)
239 }
240
241 // Get reading list
242 let readingList = []
243 for (let i = 1; i <= 6; i++) {
244 let title = frontEndApiCourseDetails[`reading_list_title_${i}`]
245 if (title && title.trim().length > 0) {
246 let reading = {
247 title: title.trim()
248 }
249
250 let author = frontEndApiCourseDetails[`reading_list_author_${i}`]
251 if (author && author.trim().length > 0) {
252 reading.author = author.trim()
253 }
254 readingList.push(reading)
255 }
256 }
257 if (readingList.length > 0) {
258 courseData.readingList = readingList
259 }
260
261 // Get prerequisites and restrictions
262 courseData.prerequisites = frontEndApiCourseDetails.other_restrictions
263
264 // Get other information
265 courseData.otherinformation = frontEndApiCourseDetails.other_information
266
267 // Get other information
268 courseData.otherrequirements = frontEndApiCourseDetails.other_requirements
269
270 // Get other information
271 courseData.website = frontEndApiCourseDetails.web_address
272
273 courseModel.createCourse(semester, subject.code, courseData, function () {
274 // Decrement the number of courses pending processing
275 coursesPendingProcessing--
276
277 // If there are no courses pending processing, we should quit
278 if (coursesPendingProcessing === 0) {
279 console.log('All courses successfully processed.')
280 process.exit()
281 }
282 })
283 })
284 }
285}
286
287// Initialise a counter of the number of courses pending being added to the database
288var coursesPendingProcessing = 0
289
290// Get queryString from command line args
291var queryString = 'term=all&subject=all'
292if (process.argv.length > 2) {
293 queryString = process.argv[2]
294}
295
296const getRegistrarFrontEndAPIToken = function (callback) {
297 request('https://registrar.princeton.edu/course-offerings', function (error, response, body) {
298 if (error) {
299 return callback(error)
300 }
301
302 const $ = cheerio.load(body)
303 const registrarFrontEndAPIToken = JSON.parse($('[data-drupal-selector="drupal-settings-json"]').text()).ps_registrar.apiToken
304 callback(null, registrarFrontEndAPIToken)
305 })
306}
307
308console.log("Acquiring API token for the registrar's website front-end API")
309
310getRegistrarFrontEndAPIToken(function (error, apiToken) {
311 if (error) {
312 console.log('Failed getting registrarFrontEndAPIToken')
313 return console.log(error)
314 }
315 console.log('Got registrarFrontEndAPIToken')
316 registrarFrontEndAPIToken = apiToken
317})
318
319// Execute a script to import courses from all available semesters ("terms") and all available departments ("subjects")
320loadCoursesFromRegistrar(queryString, importDataFromRegistrar)
321