· 8 years ago · Mar 13, 2018, 05:48 AM
1<?php
2/**
3 * Hyper Pudding
4 * - rapid and stupid search engine script for PHP source code. -
5 *
6 * @author sotarok
7 * @versoin 0.1.2
8 * @license The MIT License
9 * @requir PHP > 5.3
10 */
11
12namespace pudding\search;
13
14ini_set("memory_limit", -1);
15
16const VERSION = "0.1.2";
17
18$version = VERSION;
19echo <<<EEE
20** --------------------------------------------------------------- **
21 Hyper Pudding
22 - rapid and stupid search engine script for PHP source code. -
23 pudding\index_builder
24
25 @license The MIT License
26 @author sotarok <sotaro.k /at/ gmail.com>
27 @version {$version}
28** --------------------------------------------------------------- **
29
30EEE;
31
32class index_builder
33{
34 public $is_debug;
35 protected $_basedir = "";
36 protected $_index_filename = "";
37 protected $_source_list = array();
38
39 protected $_token_index = 1;
40 protected $_token_list = array();
41 protected $_tokenize_tokens = array(
42 T_STRING_CAST,
43 T_STRING_VARNAME,
44 T_STRING,
45 T_VARIABLE,
46 );
47 protected $_inverted_index = array();
48
49 public function __construct($basedir, Array $options = array())
50 {
51 $this->_basedir = $basedir;
52 $this->_index_filename = "._index_" . md5($this->_basedir);
53
54 $this->_options = new \ArrayObject($options);
55 $this->_tokenize_tokens = $this->get_options("tokenize_tokens", $this->_tokenize_tokens);
56 $this->is_debug = $this->get_options("is_debug", false);
57 }
58
59 public function set_options(ArrayObject $option) {
60 $this->_options = $options;
61 }
62
63 public function get_options($key, $default = "")
64 {
65 return isset($this->_options->$key) ? $this->_options->$key : $default;
66 }
67
68 public function build_index()
69 {
70 $this->_source_list = $this->crawl_recursive($this->_basedir);
71
72 $sec = microtime(true);
73 $this->tokenizer();
74 $esec = microtime(true);
75 echo "Index Built: ", $esec - $sec, PHP_EOL;
76 }
77
78 public function search($keyword)
79 {
80 if (isset($this->_inverted_index[$keyword])) {
81 return $this->scoring($this->_inverted_index[$keyword]);
82 }
83 return false;
84 }
85
86 public function scoring(&$ii)
87 {
88 $scored = array();
89 $tmp = array();
90 foreach ($ii as $k => $v) {
91 if (isset($scored[$v[0]])) {
92 $scored[$v[0]]['count']++;
93 $scored[$v[0]]['pos'][] = $v[1];
94 }
95 else {
96 $scored[$v[0]] = array(
97 'count' => 0,
98 'pos' => array($v[1]),
99 );
100 }
101 }
102 uasort($scored, function($v1, $v2) {
103 return $v1['count'] < $v2['count'];
104 });
105
106 return $scored;
107 }
108
109 public function tokenizer()
110 {
111 foreach ($this->_source_list as $source) {
112 $this->info($source, PHP_EOL);
113 foreach (token_get_all(file_get_contents($source)) as $token) {
114 if (in_array($token[0], $this->_tokenize_tokens)) {
115 $this->_token_list[] = array($source, $token[1], $token[2]);
116 if (!isset($this->_inverted_index[$token[1]])) {
117 $this->_inverted_index[$token[1]] = array();
118 }
119 $this->_inverted_index[$token[1]][] = array($source, $token[2]);
120 }
121 }
122 $this->info("\t", memory_get_usage()/1024/1024, " MB ", PHP_EOL);
123 }
124 }
125
126 public function save()
127 {
128 file_put_contents($this->filename(), serialize($this->_inverted_index));
129 return $this;
130 }
131
132 public function load()
133 {
134 $this->_inverted_index = unserialize(file_get_contents($this->filename()));
135 return $this;
136 }
137
138 public function filename($filename = null)
139 {
140 if ($filename === null) {
141 return $this->_index_filename;
142 }
143 else {
144 $this->_index_filename = $filename;
145 return $this;
146 }
147 }
148
149 public function index_file_exists()
150 {
151 return file_exists($this->filename());
152 }
153
154 public function delete_index_file()
155 {
156 if (!unlink($this->filename())) {
157 throw new \Exception(sprintf("Cannot delete index file (%s).", $this->filename()));
158 }
159 return $this;
160 }
161
162 public function info()
163 {
164 if ($this->is_debug) {
165 fprintf(STDERR, join(" ", func_get_args()));
166 }
167 }
168
169 public function crawl_recursive ($dirname)
170 {
171 $files = array();
172 foreach (glob($dirname . "/*") as $file) {
173 if (is_dir($file)) {
174 $files = array_merge($files, $this->crawl_recursive($file));
175 }
176 else {
177 if (preg_match('/.+\.(' . join($this->get_options("ext", array("php",))) .')$/', $file)) {
178 $files[] = $file;
179 }
180 }
181 }
182 return $files;
183 }
184}
185
186if ($argc != 2) {
187 fprintf(STDERR, "
188
189Invalid arguments:
190 usege: php %s base_dir
191
192 base_dir - searching source file basedir. script searcing under this directory.
193
194", $argv[0]);
195 exit(1);
196}
197
198$builder = new index_builder(rtrim($argv[1], "/"));
199
200if ($builder->index_file_exists()) {
201 echo "Indexed file found (created at ", date("Y-m-d H:i:s", filemtime($builder->filename())), ") / load ? [Y/n]:";
202 $ans = trim(fgets(STDIN));
203 if (strtolower($ans) == 'y' || empty($ans)) {
204 $builder->load();
205 }
206 else {
207 $builder->delete_index_file();
208 $builder->build_index();
209 }
210}
211else {
212 $builder->build_index();
213}
214
215echo " mem: ", printf("%.5f", memory_get_usage()/1024/1024), " MB used.", PHP_EOL;
216
217echo <<<EEE
218
219Input Search Keyword:
220 (if you want to end this script, input empty string)
221
222EEE;
223
224echo "> ";
225$key = trim(fgets(STDIN));
226while(!empty($key)) {
227 echo " searching $key", PHP_EOL;
228 $sec = microtime(true);
229 $res = $builder->search($key);
230 $esec = microtime(true);
231 if ($res) {
232 foreach ($res as $k => $r) {
233 echo " ", sprintf("%-50s", str_replace($argv[1], "", $k)), " on line ", join(", ", $r['pos']), PHP_EOL;
234 }
235 }
236 else {
237 echo "Not Found.", PHP_EOL;
238 }
239
240 echo " sec: ", printf("%.5f", $esec - $sec), PHP_EOL;
241
242 echo "> ";
243 $key = trim(fgets(STDIN));
244}
245
246echo "Do you want to save? [y/N]: ";
247$ans = trim(fgets(STDIN));
248if (strtolower($ans) == 'y') {
249 $builder->save();
250}