php与python实现的线程池多线程爬虫功能示例

php;"> url = $url; } public function run() { $ch = $this->worker->getConnection(); curl_setopt($ch,CURLOPT_URL,$this->url); $page = curl_exec($ch); $info = curl_geTinfo($ch); $error = curl_error($ch); $this->deal_data($this->url,$page,$info,$error); $this->result = $page; } function deal_data($url,$error) { $parts = explode(".",$url); $id = $parts[1]; if ($info['http_code'] != 200) { $this->show_msg($id,$error); } else { $this->show_msg($id,"OK"); } } function show_msg($id,$msg) { echo $id."\t$msg\n"; } public function getResult() { return $this->result; } protected $url; protected $result; } function check_urls_multi_pthreads() { global $check_urls; //定义抓取的连接 $check_urls = array( 'http://xxx.com' => "xx网",); $pool = new Pool(10,"Connect",array()); //建立10个线程池 foreach ($check_urls as $url => $Name) { $pool->submit(new Query($url)); } $pool->shutdown(); } check_urls_multi_pthreads(); python 多线程 def handle(sid)://这个方法内执行爬虫数据处理 pass class myThread(Thread): """docString for ClassName""" def __init__(self,sid): Thread.__init__(self) self.sid = sid def run(): handle(self.sid) threads = [] for i in xrange(1,11): t = myThread(i) threads.append(t) t.start() for t in threads: t.join()

s.get() print(url) sock = socket.socket() sock.connect(('localhost',3000)) get = 'GET {} http/1.0\r\nHost: localhost\r\n\r\n'.format(url) sock.send(get.encode('ascii')) response = b'' chunk = sock.recv(4096) while chunk: response += chunk chunk = sock.recv(4096) links = self.parse_links(url,responsE) lock.acquire() for link in links.difference(seen_urls): self.tasks.put(link) seen_urls.update(links) lock.release() self.tasks.task_done() def parse_links(self,fetched_url,responsE): if not response: print('error: {}'.format(fetched_url)) return set() if not self._is_html(responsE): return set() urls = set(re.findall(r'''(?i)href=["']?([^\s"'<>]+)''',self.body(responsE))) links = set() for url in urls: normalized = urllib.parse.urljoin(fetched_url,url) parts = urllib.parse.urlparse(normalized) if parts.scheR_564_11845@e not in ('','http','https'): conTinue host,port = urllib.parse.splitport(parts.netloC) if host and host.lower() not in ('localhost'): conTinue defragmented,frag = urllib.parse.urldefrag(parts.path) links.add(defragmented) return links def body(self,responsE): body = response.split(b'\r\n\r\n',1)[1] return body.decode('utf-8') def _is_html(self,responsE): head,body = response.split(b'\r\n\r\n',1) headers = Dict(h.split(': ') for h in head.decode().split('\r\n')[1:]) return headers.get('Content-Type','').startswith('text/html') class ThreadPool: def __init__(self,num_threads): self.tasks = Queue() for _ in range(num_threads): Fetcher(self.tasks) def add_task(self,url): self.tasks.put(url) def wait_completion(self): self.tasks.join() if __name__ == '__main__': start = time.time() pool = ThreadPool(4) pool.add_task("/") pool.wait_completion() print('{} URLs fetched in {:.1f} seconds'.format(len(seen_urls),time.time() - start))