import urllib2 from gzip import GzipFile from StringIO import StringIO class ContentEncodingProcessor(urllib2.BaseHandler): """A handler to add gzip capabilities to urllib2 requests """   # add headers to requestsdef http_request(self, req): req.add_header("Accept-Encoding", "gzip, deflate") return req   # decode defhttp_response(self, req, resp): old_resp = resp # gzip if resp.headers.get("content-encoding") == "gzip": gz = GzipFile( fileobj=StringIO(resp.read()), mode="r" ) resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg # deflate if resp.headers.get("content-encoding") == "deflate": gz =StringIO( deflate(resp.read()) ) resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) # 'class to add info() and resp.msg = old_resp.msg return resp   # deflate support import zlib def deflate(data): # zlib only provides the zlib compress format, not the deflate format; try: # so on top of all there's this workaround:return zlib.decompress(data, -zlib.MAX_WBITS) except zlib.error: return zlib.decompress(data)

encoding_support = ContentEncodingProcessor opener = urllib2.build_opener( encoding_support,urllib2.HTTPHandler )   #直接用opener打开网页，如果服务器支持gzip/defalte则自动解压缩 content = opener.open(url).read()

from twisted.web.client import getPage from twisted.internet import reactor   links = ['http://www.verycd.com/topics/%d/'%i for i in range(5420,5430) ]   def parse_page(data,url): print len(data),url  def fetch_error(error,url): print error.getErrorMessage(),url   # 批量抓取链接 for url in links: getPage(url,timeout=5)\ .addCallback(parse_page,url) \ #成功则调用parse_page方法 .addErrback(fetch_error,url) #失败则调用fetch_error方法   reactor.callLater(5, reactor.stop) #5秒钟后通知reactor结束程序 reactor.run()

f = Fetcher(threads=10) #设定下载线程数为10 for url in urls: f.push(url) #把所有url推入下载队列 whilef.taskleft(): #若还有未完成下载的线程 content = f.pop() #从下载完成队列中取出结果 do_with(content) # 处理content内容

import urllib2 from threading import Thread,Lock from Queue import Queue import time   class Fetcher: def__init__(self,threads): self.opener = urllib2.build_opener(urllib2.HTTPHandler) self.lock = Lock() #线程锁self.q_req = Queue() #任务队列 self.q_ans = Queue() #完成队列 self.threads = threads for i in range(threads): t = Thread(target=self.threadget) t.setDaemon(True) t.start() self.running = 0   def __del__(self): #解构时需等待两个队列完成 time.sleep(0.5) self.q_req.join() self.q_ans.join()   def taskleft(self): returnself.q_req.qsize()+self.q_ans.qsize()+self.running   def push(self,req): self.q_req.put(req)   def pop(self): returnself.q_ans.get()   def threadget(self): while True: req = self.q_req.get() with self.lock: #要保证该操作的原子性，进入critical area self.running += 1 try: ans = self.opener.open(req).read() except Exception, what: ans = '' printwhat self.q_ans.put((req,ans)) with self.lock: self.running -= 1 self.q_req.task_done() time.sleep(0.1) # don't spam   if __name__ == "__main__": links = [ 'http://www.verycd.com/topics/%d/'%i for i in range(5420,5430) ] f = Fetcher(threads=10) for url in links: f.push(url) while f.taskleft(): url,content = f.pop() print url,len(content)

from threading import stack_size stack_size(32768*16)

def get(self,req,retries=3): try: response = self.opener.open(req) data = response.read() except Exception , what: print what,req if retries>0: return self.get(req,retries-1) else: print 'GET Failed',req return '' return data

import socket socket.setdefaulttimeout(10) #设置10秒后连接超时

def login(self,username,password): import urllib data=urllib.urlencode({'username':username,'password':password, 'continue':'http://www.verycd.com/', 'login_submit':u'登录'.encode('utf-8'),'save_cookie':1,}) url = 'http://www.verycd.com/signin' self.opener.open(url,data).read()