Za7uM4LX

1CREATE TABLE IF NOT EXISTS `links` (
2  `link_id` int(10) unsigned NOT NULL,
3  `domain_id` mediumint(7) unsigned NOT NULL,
4  `parent_id` int(11) unsigned DEFAULT NULL,
5  `hash` int(10) unsigned NOT NULL,
6  `url` text NOT NULL,
7  `type` enum('html','pdf') DEFAULT NULL,
8  `processed` enum('N','Y') NOT NULL DEFAULT 'N',
9  UNIQUE KEY `hash` (`hash`),
10  KEY `idx_processed` (`processed`),
11  KEY `domain_id` (`domain_id`)
12) ENGINE=MyISAM DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT;
13
14
15CREATE TABLE IF NOT EXISTS `domains` (
16  `domain_id` mediumint(7) unsigned NOT NULL AUTO_INCREMENT,
17  `name` varchar(170) NOT NULL,
18  `blocked` enum('N','Y') NOT NULL DEFAULT 'N',
19  `count` mediumint(6) NOT NULL DEFAULT '0',
20  `mcount` mediumint(3) NOT NULL,
21  PRIMARY KEY (`domain_id`),
22  KEY `name` (`name`),
23  KEY `blocked` (`blocked`),
24  KEY `mcount` (`mcount`),
25  KEY `count` (`count`)
26) ENGINE=MyISAM  DEFAULT CHARSET=utf8 AUTO_INCREMENT=10834389 ;
27	
28(SELECT link_id, url, hash FROM links, domains WHERE links.domain_id = domains.domain_id and mcount > 1 and processed='N' limit 200) 
29UNION
30(SELECT link_id, url, hash FROM links where processed='N' and type='html' limit 200)
31	
32| id | select_type  | table      | type  | possible_keys           | key      |     key_len | ref                       | rows    | Extra       |
33+----+--------------+------------+-------+-------------------------+---------------        +---------+---------------------------+---------+-------------+
34|  1 | PRIMARY      | domains    | range | PRIMARY,mcount          | mcount        | 3       | NULL                      |  257673 | Using where | 
35|  1 | PRIMARY      | links      | ref   | idx_processed,domain_id | domain_id     | 3       | crawler.domains.domain_id |       1 | Using where | 
36|  2 | UNION        | links      | ref   | idx_processed           | idx_processed | 1       | const                     | 7090017 | Using where | 
37| NULL | UNION RESULT | <union1,2> | ALL   | NULL                    | NULL          | NULL    | NULL                      |    NULL |             | 
38+----+--------------+------------+-------+-------------------------+---------------+---------+---------------------------+---------+-------------+
39	
40SELECT link_id, url, hash
41FROM links, domains 
42WHERE links.domain_id = domains.domain_id 
43      AND mcount > 1
44      AND processed='N' 
45      AND type='html'