| 001 | #coding:utf-8 |
| 002 | import pycurl,re,StringIO |
| 003 | import threading,Queue,time |
| 004 | |
| 005 | class caiji: |
| 006 | #打开网页 url：网页URL |
| 007 | def html(self,url): |
| 008 | while 1: |
| 009 | try: |
| 010 | b=StringIO.StringIO() |
| 011 | c=pycurl.Curl() |
| 012 | c.setopt(pycurl.URL,url) #打开URL |
| 013 | c.setopt(pycurl.FOLLOWLOCATION,2) #允许跟踪来源，有参数：1和2 |
| 014 | c.setopt(pycurl.ENCODING, 'gzip') #开启gzip压缩提高下载速度 |
| 015 | c.setopt(pycurl.NOSIGNAL, True) #开启后多线程不会报错 |
| 016 | c.setopt(pycurl.MAXREDIRS,1) #最大重定向次数，0表示不重定向 |
| 017 | c.setopt(pycurl.CONNECTTIMEOUT,60) #链接超时 |
| 018 | c.setopt(pycurl.TIMEOUT,30) #下载超时 |
| 019 | c.setopt(pycurl.USERAGENT,'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)') |
| 020 | #pycurl.USERAGENT 模拟浏览器 |
| 021 | c.setopt(pycurl.WRITEFUNCTION, b.write) #回调写入字符串缓存 |
| 022 | c.perform() #执行上述访问网址的操作 |
| 023 | # print c.getinfo(pycurl.HTTP_CODE) |
| 024 | c.close() |
| 025 | html=b.getvalue() #读取b中的数据 |
| 026 | return html #跳出并返回html |
| 027 | except: |
| 028 | continue |
| 029 | |
| 030 | |
| 031 | wurl=open(r"url1.csv",'a') |
| 032 | |
| 033 | caiji=caiji() |
| 034 | |
| 035 | class count: |
| 036 | def __init__(self): |
| 037 | self.shoulu=0 |
| 038 | self.wshoulu=0 |
| 039 | self.i=0 |
| 040 | self.lock=threading.Lock() |
| 041 | |
| 042 | |
| 043 | def c_wshoulu(self): |
| 044 | self.lock.acquire() |
| 045 | self.wshoulu+=1 |
| 046 | wshoulu=self.wshoulu |
| 047 | self.lock.release() |
| 048 | return wshoulu |
| 049 | |
| 050 | |
| 051 | def c_sl(self): |
| 052 | self.lock.acquire() |
| 053 | self.shoulu+=1 |
| 054 | shoulu=self.shoulu |
| 055 | self.lock.release() |
| 056 | return shoulu |
| 057 | |
| 058 | |
| 059 | def c_i(self): |
| 060 | self.lock.acquire() |
| 061 | self.i+=1 |
| 062 | i=self.i |
| 063 | self.lock.release() |
| 064 | return i |
| 065 | |
| 066 | count=count() |
| 067 | |
| 068 | class th(threading.Thread): |
| 069 | def __init__(self,qurl): |
| 070 | threading.Thread.__init__(self) |
| 071 | self.qurl=qurl |
| 072 | self.lock=threading.Lock() |
| 073 | self.cond=threading.Condition() |
| 074 | |
| 075 | |
| 076 | def run(self): |
| 077 | while 1: |
| 078 | ddc=self.qurl.get() |
| 079 | if ddc is None: |
| 080 | break |
| 081 | while 1: |
| 082 | bdhtm=caiji.html('http://www.baidu.com/s?wd='+ddc) |
| 083 | self.lock.acquire() |
| 084 | |
| 085 | if '百度为您找到相关结果约' in bdhtm: |
| 086 | i=count.c_i() |
| 087 | print '第%s条, %s ,收录'% (i,ddc) |
| 088 | wurl.writelines('第%s条, %s ,收录\n'% (i,ddc)) |
| 089 | count.c_sl() |
| 090 | break |
| 091 | |
| 092 | elif '抱歉，没有找到与' in bdhtm: |
| 093 | i=count.c_i() |
| 094 | print '第%s条, %s ,未收录'% (i,ddc) |
| 095 | wurl.writelines('第%s条, %s ,未收录\n'% (i,ddc)) |
| 096 | count.c_wshoulu() |
| 097 | break |
| 098 | |
| 099 | elif 'http://verify.baidu.com/' in bdhtm: |
| 100 | print ddc,'出现验证码，等待5分钟后自动开始' |
| 101 | self.lock.release() |
| 102 | time.sleep(500) |
| 103 | continue |
| 104 | |
| 105 | else: |
| 106 | print 'Error' |
| 107 | break |
| 108 | self.lock.release() |
| 109 | |
| 110 | |
| 111 | qurl=Queue.Queue(0) |
| 112 | threadCount=6 #开启线程数，默认6个线程 |
| 113 | |
| 114 | ths=[] |
| 115 | for t in range(threadCount): |
| 116 | thread=th(qurl) |
| 117 | thread.start() |
| 118 | ths.append(thread) |
| 119 | |
| 120 | for ddc in open(r'url.csv'): #导入需要查询的URL文件，格式必须是utf-8 |
| 121 | ddc=ddc[0:-1] |
| 122 | qurl.put(ddc) |
| 123 | |
| 124 | for tt in range(threadCount): |
| 125 | qurl.put(None) |
| 126 | |
| 127 | for t in ths: |
| 128 | t.join() |
| 129 | |
| 130 | sl=count.c_sl()-1 |
| 131 | |
| 132 | print '\n收录率：'+str(round(float(sl)/float(count.c_i()-1)*100,2)),"%" |
| 133 | print '收录：%s 条'%str(sl) |
| 134 | print '未收录：%s 条'%str(count.c_wshoulu()-1) |
| 135 | |
| 136 | |
| 137 | |
