· 8 years ago · Feb 08, 2018, 03:12 PM
1<?php
2// ** Features: **
3// ** - Logs in to the forum, so no unaccessable threads **
4// ** - Iterates through all threads within a given range **
5// ** - Will skip missing/deleted threads automatically **
6// ** - Generates an XML file for each thread, including: **
7// ** - Thread Topic **
8// ** - Post Author **
9// ** - Post Date/Time **
10// ** - Details of any externally linked images **
11// ** - Details of any quoted posts, including content **
12// ************************************************************
13// ** Edit the following details to suit **
14// ************************************************************
15// Username and Forum details
16$username = "username";
17$password = "password";
18$url = "http://preludeuk.forumup.com";
19// this is the name of the folder to save the files
20$projectname = "preludeuk";
21// Select the topics you wish to grab
22$start_topic = 1;
23$end_topic = 10000;
24// optionally add a delay (in milliseconds) between requests to
25// prevent problems if a site throttles you
26$delay = 0;
27// verbose messages during run?
28$verbose = false;
29
30// ************************************************************
31// * BEYOND THIS LIES THE MAIN CODE, EDIT AT YOUR OWN RISK! *
32// ************************************************************
33// ------------------------------------------------------------
34// This part logs you in to the forum!
35// ------------------------------------------------------------
36// Set login URL
37$login_url = $url . "/login.php";
38// new curl object
39$ch = curl_init();
40// set Curl Options
41curl_setopt($ch, CURLOPT_URL, $login_url);
42curl_setopt($ch, CURLOPT_POST, 1);
43$post_fields = array(
44 'username' => $username,
45 'password' => $password,
46 'autologin' => 0,
47 'redirect' => 'index.php',
48 'login' => 'Log In',
49);
50curl_setopt($ch, CURLOPT_POSTFIELDS, $post_fields);
51curl_setopt($ch, CURLOPT_COOKIEJAR, "cookie.txt");
52curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
53// execute curl,fetch the result and close curl connection
54$result = curl_exec($ch);
55// check for errors - if we can't login, exit!
56if (curl_errno($ch)){
57 die (curl_errno($ch));
58}
59
60// ------------------------------------------------------------
61// This part grabs the topics, posts, and images
62// ------------------------------------------------------------
63
64// Set topic and page
65$topic = $start_topic;
66$page = 1;
67$topic_content = "";
68$topic_found = false;
69$first_post = true;
70$seen_postbody = false;
71
72// topic check, stops us printing the title each time round the loop
73$topic_check = 0;
74
75// Create a directory to hold the xml files
76// sanitise the project name to do so
77$foldername = preg_replace("/[^A-Za-z0-9]/","",strtolower(trim($projectname)));
78@mkdir($foldername);
79
80// outer loop
81while ($topic <= $end_topic) {
82
83 // set whether or not we wish to grab any data
84 $good_to_go = false;
85
86 // Calculate offset (Forum pages show 15 posts at a time)
87 $offset = 15*($page - 1);
88
89 // Create the url
90 $url_to_try = $url . '/viewtopic.php?t=' . $topic . '&start=' . $offset;
91
92 // Set the URL of the page
93 curl_setopt($ch, CURLOPT_URL, $url_to_try);
94
95 // we want to check what we get back
96 curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
97
98 // grab the reply
99 log_print("grabbing url $url_to_try");
100 $result = curl_exec($ch);
101
102
103 // Check if we have an error message meaning no topic found
104 if (strstr($result, "No posts exist for this topic") !== false) {
105 // If we have something worth writing
106 if (strlen($topic_content) > 0) {
107 log_print("writing content for topic: $current_title");
108 // close the XML tags
109 $topic_content .= "</thread>\n";
110 // We have dropped off the end of the topic, save what we have
111 $file_name = $foldername . "/topic_" . str_pad($topic,6,"0",STR_PAD_LEFT) . "_" . preg_replace("/[^A-Za-z0-9_]/","",strtolower(trim(substr(str_replace(" ", "_", $current_title),0,30)))) . ".xml";
112 $fh = fopen($file_name, 'w') or die("ERROR: Can't open file: " . $file_name);
113 fwrite($fh, $topic_content);
114 fclose($fh);
115 }
116 // Topic is missing, increment the counter and reset variables
117 $good_to_go = false;
118 $topic_found = false;
119 $topic++;
120 $page = 1;
121 $topic_content = "";
122 $current_title = "";
123 } else {
124 // The below means we are too many pages into the topic - we've dropped off the end, time to find a new topic!
125 if (strstr($result, "The topic or post you requested does not exist") !== false) {
126 // If we have something worth writing
127 if (strlen($topic_content) > 0) {
128 log_print("writing content for topic: $current_title");
129 // close xml tags
130 $topic_content .= "</thread>\n";
131 // We have dropped off the end of the topic, save what we have
132 $file_name = $foldername . "/topic_" . str_pad($topic,6,"0",STR_PAD_LEFT) . "_" . preg_replace("/[^A-Za-z0-9_]/","",strtolower(trim(substr(str_replace(" ", "_", $current_title),0,30)))) . ".xml";
133 $fh = fopen($file_name, 'w') or die("ERROR: Can't open file: " . $file_name);
134 fwrite($fh, $topic_content);
135 fclose($fh);
136 }
137 // Now increment the topic and reset the variables
138 $topic_content = "";
139 $current_title = "";
140 $topic++;
141 $page = 1;
142 } else {
143 if ($topic_check != $topic) {
144 // New topic, so we need to change this to match
145 log_print("New topic: $topic");
146 $topic_check = $topic;
147 }
148 // If we are here, we *do* want to harvest data
149 log_print("Good go to, ready to harvest data");
150 $good_to_go = true;
151 }
152 }
153
154 // if we have a valid page hit, harvest the details!
155 if ($good_to_go) {
156 // Clean up returned html
157 $result = @mb_convert_encoding($result, 'HTML-ENTITIES', 'utf-8');
158 // New DOM document
159 $dom = new DOMDocument();
160 @$dom->loadHTML($result);
161 // grab page 'nodes' (tags basically)
162 $nodes = $dom->getElementsByTagName('*');
163 // now loop through the tags and grab name and post content
164 log_print("parsing data from posts");
165 foreach($nodes as $node) {
166 //Get Topic
167 if($node->nodeName == 'a' && $node->getAttribute('class') == 'maintitle' && !$topic_found) {
168 $topic_content .= "<?xml version='1.0' encoding='UTF-8' ?>\n";
169 $topic_content .= "<thread>\n";
170 $topic_content .= "\t<database_id>" . $topic . "</database_id>\n";
171 $topic_content .= "\t<title>" . htmlentities($node->nodeValue, ENT_COMPAT, "UTF-8") . "</title>\n";
172 $topic_found = true;
173 $current_title = $node->nodeValue;
174 }
175 // Get Post Owner
176 if($node->nodeName == 'span' && $node->getAttribute('class') == 'name') {
177 if(!$first_post) {
178 // Close post
179 $topic_content .= "\t\t</message>\n";
180 $topic_content .= "\t</post>\n";
181 }
182 // Start new post
183 $seen_postbody = false;
184 $topic_content .= "\t<post>\n";
185 $topic_content .= "\t\t<author>" . htmlentities($node->nodeValue, ENT_COMPAT, "UTF-8") . "</author>\n";
186 // first post is grabbed
187 $first_post = false;
188 }
189 // Get Post time/date
190 if($node->nodeName == 'span' && $node->getAttribute('class') == 'postdetails' && (strstr($node->nodeValue, "Posted:") !== false)) {
191 // We need to strip the content down to grab just the time/date
192 $stripstring = trim(substr($node->nodeValue,8,25));
193 $stripstring = htmlentities(utf8_encode($stripstring));
194 $stripstring = str_replace(array("Â"," "), "", $stripstring);
195 $topic_content .= "\t\t<time>" . $stripstring . "</time>\n";
196 $topic_content .= "\t\t<message>\n";
197 }
198 // Get Post Content
199 if($node->nodeName == 'span' && $node->getAttribute('class') == 'postbody' && !$seen_postbody) {
200 log_print("Processing post: " . $node->nodeValue);
201 // We've found the post body!
202 $seen_postbody = true;
203 // Check if we have extra data in the post
204 if ($node->hasChildNodes()) {
205 $inodes = $node->childNodes;
206 foreach($inodes as $inode) {
207 // If we have an image - check they are external, and not smilies
208 if($inode->nodeName == 'img' && strstr($inode->getAttribute('src'), "http") !== false) {
209 $topic_content .= "\t\t\t<linkedimage>" . htmlentities($inode->getAttribute('src'), ENT_COMPAT, "UTF-8") . "</linkedimage>\n";
210 }
211 if($inode->nodeName == 'div') {
212 $iinodes = $inode->childNodes;
213 foreach($iinodes as $iinode) {
214 // If we have a table, it should contain a quote
215 if($iinode->nodeName == 'table' && $iinode->hasChildNodes()) {
216 // recurse to grab quotes
217 $topic_content .= print_quotes($iinode, 1);
218 // remove recursed content
219 remove_children($iinode);
220 }
221 }
222 }
223 }
224 }
225 // Remove signature if it exists
226 $message_body = htmlentities($node->nodeValue, ENT_COMPAT, "UTF-8");
227 if(strstr($message_body, "_________________") !== false) {
228 $message_body = substr($message_body,0,strpos($message_body,"_________________"));
229 }
230
231 // Remove any children and print main message
232 $topic_content .= preg_replace('/(\r\n|\r|\n)/s',"\n",$message_body) . "\n";
233 }
234 }
235
236 // Close last post
237 $topic_content .= "\t\t</message>\n";
238 $topic_content .= "\t</post>\n";
239
240 // Update page count and back round the loop
241 $page++;
242 }
243
244 if ($delay > 0) {
245 usleep($delay * 1000);
246 }
247
248}
249
250// Close cURL session
251curl_close($ch);
252
253// *****************************************************************
254// Functions Required for the Script
255// *****************************************************************
256
257// This recurses and prints all quotes
258function print_quotes($node, $depth) {
259 // Start with empty topic string
260 $topic_content = "";
261 // Increase level as we go deeper
262 $depth++;
263 // default tab level
264 $quotetab = "\t\t";
265 $detailtab = "\t\t\t";
266 // set tab indentation based on level
267 for($i=1; $i < $depth; $i++) {
268 $quotetab .= "\t";
269 $detailtab .= "\t";
270 }
271 // now start again
272 $inodes = $node->childNodes;
273 foreach($inodes as $inode) {
274 // Drill down to tr
275 if ($inode->nodeName == 'tr' && $inode->hasChildNodes()) {
276 $iinodes = $inode->childNodes;
277 foreach($iinodes as $iinode) {
278 // Drill down to td
279 if ($iinode->nodeName == 'td' && $iinode->hasChildNodes()) {
280 $iiinodes = $iinode->childNodes;
281 foreach($iiinodes as $iiinode) {
282 // And now catch any quotes
283 if($iiinode->nodeName == 'span' && $iiinode->getAttribute('class') == 'genmed') {
284 // We have found a quote
285 // Get the author
286 $topic_content .= $quotetab . "<quote>\n";
287 $topic_content .= $detailtab . "<author>" . htmlentities(str_replace(" wrote:", "", $iiinode->nodeValue), ENT_COMPAT, "UTF-8") . "</author>\n";
288 $topic_content .= $detailtab . "<message>\n";
289 }
290 }
291 }
292 if($iinode->nodeName == 'td' && $iinode->getAttribute('class') == 'quote') {
293 // Check for nested quotes
294 if ($iinode->hasChildNodes()) {
295 $iiinodes = $iinode->childNodes;
296 foreach($iiinodes as $iiinode) {
297 if($iiinode->nodeName == 'table' && $iiinode->hasChildNodes()) {
298 // Recurse!
299 $topic_content .= print_quotes($iiinode, $depth);
300 // remove recursed content
301 remove_children($iiinode);
302 }
303 }
304 }
305 // Get content
306 $topic_content .= preg_replace('/(\r\n|\r|\n)/s',"\n",htmlentities($iinode->nodeValue, ENT_COMPAT, "UTF-8")) . "\n";
307 $topic_content .= $detailtab . "</message>\n";
308 $topic_content .= $quotetab . "</quote>\n";
309 }
310 }
311 }
312 }
313 // Return all the recursing goodness
314 return $topic_content;
315}
316
317// This removes all 'childnodes' - required to give us clean message
318// data. Required as we are splitting quotes out
319function remove_children(&$node) {
320 while ($node->firstChild) {
321 while ($node->firstChild->firstChild) {
322 remove_children($node->firstChild);
323 }
324 $node->removeChild($node->firstChild);
325 }
326}
327
328function log_print($log_text) {
329 global $verbose;
330 if ($verbose) {
331 $timestamp = date('Y-m-d H:i:s');
332 echo "[$timestamp] $log_text\n";
333 }
334}
335?>