#coding:utf-8import urllib2,re,requests
from bs4 import BeautifulSoup as bsdomain = 'www.123.com'    #要查询的域名
page_num = 10 * 10     #第一个数字为要抓取的页数def gethtml(url):headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',# 'Accept-Encoding':'gzip, deflate, sdch','Accept-Language':'zh-CN,zh;q=0.8','Cache-Control':'max-age=0','Connection':'keep-alive','Cookie':'BDUSS=ng4UFVyUUpWU2hUR2R3b3hKamtpaE9ocW40LTFZcGdWeDBjbXkzdE83eDJQSE5YQVFBQUFBJCQAAAAAAAAAAAEAAADD3IYSamFjazE1NDUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHavS1d2r0tXa; ispeed_lsm=2; PSTM=1465195705; BIDUPSID=2274339847BBF9B1E97DA3ECE6469761; H_WISE_SIDS=102907_106764_106364_101556_100121_102478_102628_106368_103569_106502_106349_106665_106589_104341_106323_104000_104613_104638_106071_106599_106795; BAIDUID=D94A8DE66CF701AB5C3332B1BF883DDC:FG=1; BDSFRCVID=UEusJeC62m80hjJRoxzDhboaBeKaL6vTH6aIa6lTlb9Zx-72yRF7EG0PfOlQpYD-d1GyogKK3gOTH4jP; H_BDCLCKID_SF=fR-foIPbtKvSq5rvKbOEhPCX-fvQh4JXHD7yWCvG3455OR5Jj65Ve58JM46N2bvE3IbaWbjP5lvH8KQC3MA--fF_jxvn2PD8yj-L_KoXLqLbsq0x0-jchh_QWt8LKToxMCOMahkb5h7xOKbF056jK4JKjH0qt5cP; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a02157232777; BD_HOME=1; BD_UPN=12314353; sug=3; sugstore=1; ORIGIN=0; bdime=0; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; H_PS_645EC=a5cfUippkbo0uQPU%2F4QbUFVCqXu4W9g5gr5yrxTnJT10%2FElVEvJBbeyjWJq8QUHgepjd; BD_CK_SAM=1; BDSVRTM=323; H_PS_PSSID=1434_20317_12896_20076_19860_17001_15506_11866; __bsi=16130066511508055252_00_0_I_R_326_0303_C02F_N_I_I_0',# 'Host':'www.baidu.com','Upgrade-Insecure-Requests':'1','User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36',}req = urllib2.Request(url=url,headers=headers)html = urllib2.urlopen(req,timeout = 30).read()return htmldef status(url):    #返回状态码status = requests.get(url).status_codereturn statusstatus_file = open('url_status.txt','a+')
for i in range(10,page_num,10):url = 'https://www.baidu.com/s?wd=site%3A' + domain + '&pn=' + str(i)html = gethtml(url)soup = bs(html,"lxml")for i in soup.select('.c-showurl'):# print i.get('href')urls = i.get('href')# url_list.append(urls)    header = requests.head(urls).headersheader_url = header['location']    #获取真实URLif int(status(header_url)) == 404:print status(header_url),header_url    #打印状态码和真实URLstatus_file.write(str(status(header_url)) + ' ' + header_url + '\n')    #获取的状态码和链接写入文件
status_file.close()
#获取状态码函数

#coding: utf-8
import sys
import urllib
import urllib2
from BeautifulSoup import BeautifulSoupquestion_word = "吃货 程序员"
url = "http://www.baidu.com/s?wd=" + urllib.quote(question_word.decode(sys.stdin.encoding).encode('gbk'))
htmlpage = urllib2.urlopen(url).read()
soup = BeautifulSoup(htmlpage)
print len(soup.findAll("table", {"class": "result"}))
for result_table in soup.findAll("table", {"class": "result"}):a_click = result_table.find("a")print "-----标题----\n" + a_click.renderContents()#标题print "----链接----\n" + str(a_click.get("href"))#链接print "----描述----\n" + result_table.find("div", {"class": "c-abstract"}).renderContents()#描述print