鏌ヨ鏀跺綍鏁帮紝鏀跺綍鐜囷紝鏈敹褰曢摼鎺ワ紝鏀跺綍鐨勯摼鎺?br/> 闇?瑕佸畨瑁卲ycurl妯″潡
闇?瑕佹煡璇㈢殑URL鏀惧湪url.csv閲岄潰鏂囦欢蹇呴』鏄痷tf-8鏍煎紡
杩愯BDshoulu.py鏂囦欢
鍦╓indows涓嬮潰鐨勫懡浠ゆ彁绀虹涓嬭繍琛屼細涔辩爜锛宲rint鐨勫唴瀹逛細涔辩爜锛岃鑷杞爜锛屼笉褰卞搷缁撴灉銆?br/> 閬囧埌楠岃瘉鐮佺殑鏃跺?欎細鍋滄5鍒嗛挓閲嶆柊鏌?br/> 绾跨▼寤鸿涓嶈寮?澶锛屽惁鍒欎細瀵艰嚧灏両P

 Python | 
 copy code |
?
 
001
#coding:utf-8
002
import pycurl,re,StringIO
003
import  threading,Queue,time
004
 
005
class caiji:
006
	#鎵撳紑缃戦〉  url锛氱綉椤礥RL
007
	def html(self,url):
008
		while 1:
009
			try:
010
				b=StringIO.StringIO()
011
				c=pycurl.Curl()
012
				c.setopt(pycurl.URL,url) #鎵撳紑URL
013
				c.setopt(pycurl.FOLLOWLOCATION,2) #鍏佽璺熻釜鏉ユ簮锛屾湁鍙傛暟锛??
014
				c.setopt(pycurl.ENCODING, 'gzip')  #寮?鍚痝zip鍘嬬缉鎻愰珮涓嬭浇閫熷害
015
				c.setopt(pycurl.NOSIGNAL, True)   #寮?鍚悗澶氱嚎绋嬩笉浼氭姤閿?/span>
016
				c.setopt(pycurl.MAXREDIRS,1) #鏈?澶ч噸瀹氬悜娆℃暟锛?琛ㄧず涓嶉噸瀹氬悜
017
				c.setopt(pycurl.CONNECTTIMEOUT,60) #閾炬帴瓒呮椂
018
				c.setopt(pycurl.TIMEOUT,30)  #涓嬭浇瓒呮椂
019
				c.setopt(pycurl.USERAGENT,'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)')
020
				#pycurl.USERAGENT  妯℃嫙娴忚鍣?/span>
021
				c.setopt(pycurl.WRITEFUNCTION, b.write)  #鍥炶皟鍐欏叆瀛楃涓茬紦瀛?/span>
022
				c.perform() #鎵ц涓婅堪璁块棶缃戝潃鐨勬搷浣?/span>
023
				# print c.getinfo(pycurl.HTTP_CODE)
024
				c.close()
025
				html=b.getvalue()   #璇诲彇b涓殑鏁版嵁
026
				return html    #璺冲嚭骞惰繑鍥瀐tml
027
			except:
028
				continue
029
 
030
 
031
wurl=open(r"url1.csv",'a')
032
 
033
caiji=caiji()
034
 
035
class count:
036
	def __init__(self):
037
		self.shoulu=0
038
		self.wshoulu=0
039
		self.i=0
040
		self.lock=threading.Lock()
041
 
042
 
043
	def c_wshoulu(self):
044
		self.lock.acquire()
045
		self.wshoulu+=1
046
		wshoulu=self.wshoulu
047
		self.lock.release()
048
		return wshoulu
049
 
050
 
051
	def c_sl(self):
052
		self.lock.acquire()
053
		self.shoulu+=1
054
		shoulu=self.shoulu
055
		self.lock.release()
056
		return shoulu
057
 
058
 
059
	def c_i(self):
060
		self.lock.acquire()
061
		self.i+=1
062
		i=self.i
063
		self.lock.release()
064
		return i
065
 
066
count=count()
067
 
068
class th(threading.Thread):
069
	def __init__(self,qurl):
070
		threading.Thread.__init__(self)
071
		self.qurl=qurl
072
		self.lock=threading.Lock()
073
		self.cond=threading.Condition()
074
 
075
 
076
	def run(self):
077
		while 1:
078
			ddc=self.qurl.get()
079
			if ddc is  None:
080
				break
081
			while 1:
082
				bdhtm=caiji.html('http://www.baidu.com/s?wd='+ddc)
083
				self.lock.acquire()
084
 
085
				if '鐧惧害涓烘偍鎵惧埌鐩稿叧缁撴灉绾? in bdhtm:
086
					i=count.c_i()
087
					print '绗?s鏉? %s ,鏀跺綍'% (i,ddc)
088
					wurl.writelines('绗?s鏉? %s ,鏀跺綍\n'% (i,ddc))
089
					count.c_sl()
090
					break
091
 
092
				elif '鎶辨瓑锛屾病鏈夋壘鍒颁笌' in bdhtm:
093
					i=count.c_i()
094
					print '绗?s鏉? %s ,鏈敹褰?% (i,ddc)
095
					wurl.writelines('绗?s鏉? %s ,鏈敹褰?span style="color: #000099; font-weight: bold;">\n'% (i,ddc))
096
					count.c_wshoulu()
097
					break
098
 
099
				elif 'http://verify.baidu.com/' in bdhtm:
100
					print ddc,'鍑虹幇楠岃瘉鐮侊紝绛夊緟5鍒嗛挓鍚庤嚜鍔ㄥ紑濮?
101
					self.lock.release()
102
					time.sleep(500)
103
					continue
104
 
105
				else:
106
					print 'Error'
107
					break
108
			self.lock.release()
109
 
110
 
111
qurl=Queue.Queue(0)
112
threadCount=6    #寮?鍚嚎绋嬫暟锛岄粯璁?嚎绋?/span>
113
 
114
ths=[]
115
for t in range(threadCount):
116
	thread=th(qurl)
117
	thread.start()
118
	ths.append(thread)
119
 
120
for ddc in open(r'url.csv'):   #瀵煎叆闇?瑕佹煡璇㈢殑URL鏂囦欢锛屾牸寮忓繀椤绘槸utf-8
121
	ddc=ddc[0:-1]
122
	qurl.put(ddc)
123
 
124
for tt in range(threadCount):
125
	qurl.put(None)
126
 
127
for t in ths:
128
	t.join()
129
 
130
sl=count.c_sl()-1
131
 
132
print  '\n鏀跺綍鐜囷細'+str(round(float(sl)/float(count.c_i()-1)*100,2)),"%"
133
print '鏀跺綍锛?s 鏉?%str(sl)
134
print '鏈敹褰曪細%s 鏉?%str(count.c_wshoulu()-1)
135
 
136
 
137