· 9 years ago · Jul 04, 2017, 11:34 AM
1$queryRes = mysql_query("SELECT url FROM tablName LIMIT 50000"); // everytime I am using LIMIT
2while ($row = mysql_fetch_object($queryRes)) {
3 $info = pathinfo($row->url);
4 $fileName = $info['filename'];
5 $fileExtension = $info['extension'];
6
7 try {
8 copy("http:".$row->url, "img/$fileName"."_".$row->id.".".$fileExtension);
9 } catch(Exception $e) {
10 echo "<br/>n unable to copy '$fileName'. Error:$e";
11 }
12}
13
14function getimg($url) {
15 $headers[] = 'Accept: image/gif, image/x-bitmap, image/jpeg, image/pjpeg';
16 $headers[] = 'Connection: Keep-Alive';
17 $headers[] = 'Content-type: application/x-www-form-urlencoded;charset=UTF-8';
18 $user_agent = 'php';
19 $process = curl_init($url);
20 curl_setopt($process, CURLOPT_HTTPHEADER, $headers);
21 curl_setopt($process, CURLOPT_HEADER, 0);
22 curl_setopt($process, CURLOPT_USERAGENT, $useragent);
23 curl_setopt($process, CURLOPT_TIMEOUT, 30);
24 curl_setopt($process, CURLOPT_RETURNTRANSFER, 1);
25 curl_setopt($process, CURLOPT_FOLLOWLOCATION, 1);
26 $return = curl_exec($process);
27 curl_close($process);
28 return $return;
29}
30
31$queryRes = mysql_query("select url from tablName limit 50000"); //everytime i am using limit
32while($row = mysql_fetch_object($queryRes)) {
33
34 $info = pathinfo($row->url);
35 $fileName = $info['filename'];
36 $fileExtension = $info['extension'];
37
38 if (!@copy("http:".$row->url, "img/$fileName"."_".$row->id.".".$fileExtension)) {
39 $errors= error_get_last();
40 echo "COPY ERROR: ".$errors['type'];
41 echo "<br />n".$errors['message'];
42 //you can add what ever code you wnat here... out put to conselo, log in a file put an exit() to stop dowloading...
43 }
44}
45
46if( false === file_get_contents(...) )
47 trigger_error(...);
48
49//only fetch 50 urls each time
50$queryRes = mysql_query ( "select id, url from tablName where flag=1 limit 50" );
51
52//just prefer absolute path
53$imgDirPath = dirname ( __FILE__ ) + '/';
54
55while ( $row = mysql_fetch_object ( $queryRes ) )
56{
57 $info = pathinfo ( $row->url );
58 $fileName = $info ['filename'];
59 $fileExtension = $info ['extension'];
60
61 //url in the table is like //www.example.com???
62 $result = fetchUrl ( "http:" . $row->url,
63 $imgDirPath + "img/$fileName" . "_" . $row->id . "." . $fileExtension );
64
65 if ($result !== true)
66 {
67 echo "<br/>n unable to copy '$fileName'. Error:$result";
68 //update flag to 3, finish this func yourself
69 set_row_flag ( 3, $row->id );
70 }
71 else
72 {
73 //update flag to 3
74 set_row_flag ( 2, $row->id );
75 }
76}
77
78function fetchUrl($url, $saveto)
79{
80 $ch = curl_init ( $url );
81
82 curl_setopt ( $ch, CURLOPT_FOLLOWLOCATION, true );
83 curl_setopt ( $ch, CURLOPT_MAXREDIRS, 3 );
84 curl_setopt ( $ch, CURLOPT_HEADER, false );
85 curl_setopt ( $ch, CURLOPT_RETURNTRANSFER, true );
86 curl_setopt ( $ch, CURLOPT_CONNECTTIMEOUT, 7 );
87 curl_setopt ( $ch, CURLOPT_TIMEOUT, 60 );
88
89 $raw = curl_exec ( $ch );
90
91 $error = false;
92
93 if (curl_errno ( $ch ))
94 {
95 $error = curl_error ( $ch );
96 }
97 else
98 {
99 $httpCode = curl_getinfo ( $ch, CURLINFO_HTTP_CODE );
100
101 if ($httpCode != 200)
102 {
103 $error = 'HTTP code not 200: ' . $httpCode;
104 }
105 }
106
107 curl_close ( $ch );
108
109 if ($error)
110 {
111 return $error;
112 }
113
114 file_put_contents ( $saveto, $raw );
115
116 return true;
117}
118
119$queryRes = mysql_query("SELECT id, url FROM tablName ORDER BY id");
120while (($row = mysql_fetch_object($queryRes)) !== false) {
121 $info = pathinfo($row->url);
122 $fn = $info['filename'];
123 if (copy(
124 'http:' . $row->url,
125 "img/{$fn}_{$row->id}.{$info['extension']}"
126 )) {
127 echo "success: $fnn";
128 } else {
129 echo "fail: $fnn";
130 }
131 flush();
132}
133
134CREATE TABLE IF NOT EXISTS `images` (
135 `id` int(60) NOT NULL AUTO_INCREMENTh,
136 `link` varchar(1024) NOT NULL,
137 `status` enum('not fetched','fetched') NOT NULL DEFAULT 'not fetched',
138 `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
139 PRIMARY KEY (`id`)
140);
141
142<?php
143// how many images to download in one go?
144$limit = 100;
145/* if set to true, the scraper reloads itself. Good for running on localhost without cron job support. Just keep the browser open and the script runs by itself ( javascript is needed) */
146$reload = false;
147// to prevent php timeout
148set_time_limit(0);
149// db connection ( you need pdo enabled)
150 try {
151 $host = 'localhost';
152 $dbname= 'mydbname';
153 $user = 'root';
154 $pass = '';
155 $DBH = new PDO("mysql:host=$host;dbname=$dbname", $user, $pass);
156 }
157 catch(PDOException $e) {
158 echo $e->getMessage();
159 }
160$DBH->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION );
161
162// get n number of images that are not fetched
163$query = $DBH->prepare("SELECT * FROM images WHERE status = 'not fetched' LIMIT {$limit}");
164$query->execute();
165$files = $query->fetchAll();
166// if no result, don't run
167if(empty($files)){
168 echo 'All files have been fetched!!!';
169 die();
170}
171// where to save the images?
172$savepath = dirname(__FILE__).'/scrapped/';
173// fetch 'em!
174foreach($files as $file){
175 // get_url_content uses curl. Function defined later-on
176 $content = get_url_content($file['link']);
177 // get the file name from the url. You can use random name too.
178 $url_parts_array = explode('/' , $file['link']);
179 /* assuming the image url as http:// abc . com/images/myimage.png , if we explode the string by /, the last element of the exploded array would have the filename */
180 $filename = $url_parts_array[count($url_parts_array) - 1];
181 // save fetched image
182 file_put_contents($savepath.$filename , $content);
183 // did the image save?
184 if(file_exists($savepath.$file['link']))
185 {
186 // yes? Okay, let's save the status
187 $query = $DBH->prepare("update images set status = 'fetched' WHERE id = ".$file['id']);
188 // output the name of the file that just got downloaded
189 echo $file['link']; echo '<br/>';
190 $query->execute();
191 }
192}
193
194// function definition get_url_content()
195function get_url_content($url){
196 // ummm let's make our bot look like human
197 $agent= 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)';
198 $ch = curl_init();
199 curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
200 curl_setopt($ch, CURLOPT_VERBOSE, true);
201 curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
202 curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1);
203 curl_setopt($ch, CURLOPT_USERAGENT, $agent);
204 curl_setopt($ch, CURLOPT_URL,$url);
205 return curl_exec($ch);
206}
207//reload enabled? Reload!
208if($reload)
209 echo '<script>location.reload(true);</script>';