#!/usr/bin/python

#-*-coding:utf-8-*-

# 简易采集爬虫

#    1.采集Yahoo!Answers，parseData函数修改一下，可以采集任何网站

#    2.需要sqlite3或者pysqlite支持

#    3.可以在DreamHost.com空间上面运行

#    4.可以修改User-Agent冒充搜索引擎蜘蛛

#    5.可以设置暂停的时间，控制采集速度

#    6.采集Yahoo会被封IP数小时，所以这个采集用处不大

# Author: Lukin

# Date  : 2008-09-25

# 导入采集需要用到的模块

import re, sys, time

import httplib, os.path as osp

from urlparse import urlparse

# 使用sqite数据库，为了兼容DreamHost.com的空间，只能这么写了

try :

import sqlite3 as sqlite

except ImportError:

from pysqlite2 import dbapi2 as sqlite

# 采集速度控制，单位秒

sleep = 0

# 数据库路径

dbname = './database.db'

# 设置提交的header头

headers = {"Accept": "*/*","Referer": "http://answers.yahoo.com/","User-Agent": "Mozilla/5.0+(compatible;+Googlebot/2.1;++http://www.google.com/bot.html)"}

# 连接服务器

dl = httplib.HTTPConnection('answers.yahoo.com')

# 连接数据库

conn = sqlite.connect(osp.abspath(dbname))

# 创建数据库

def createDatabase():

global conn,dbname;

if osp.isfile(osp.abspath(dbname)) : return

c = conn.cursor()

# 创建url列表存放表

c.execute('''CREATE TABLE IF NOT EXISTS [collect]([cid] INTEGER PRIMARY KEY,[curl] TEXT,[state] INTEGER DEFAULT '0',UNIQUE([curl]));''')

c.execute('''CREATE INDEX IF NOT EXISTS [collect_idx_state] ON [collect]([state]);''')

# 创建分类表

c.execute('''CREATE TABLE IF NOT EXISTS [sorts]([sortid] INTEGER PRIMARY KEY,[sortname] TEXT,[sortpath] TEXT,[sortfoot] INTEGER DEFAULT '0',[sortnum] INTEGER DEFAULT '0',UNIQUE([sortpath]));''')

c.execute('''CREATE INDEX IF NOT EXISTS [sorts_idx_sortname] ON [sorts]([sortname]);''')

c.execute('''CREATE INDEX IF NOT EXISTS [sorts_idx_sortfoot] ON [sorts]([sortfoot]);''')

# 创建文章表

c.execute('''CREATE TABLE IF NOT EXISTS [article]([aid] INTEGER PRIMARY KEY,[sortid] INTEGER DEFAULT '0',[hits] INTEGER DEFAULT '0',[title] TEXT,[path] TEXT,[question] TEXT,[banswer] TEXT,[oanswer] TEXT,UNIQUE([path]));''')

c.execute('''CREATE INDEX IF NOT EXISTS [article_idx_sortid] ON [article]([sortid]);''')

# 事物提交

conn.commit()

c.close()

# 执行采集

def collect(url="http://answers.yahoo.com/"):

global dl,error,headers; R = 0

print "GET:",url

urls = urlparse(url); path = urls[2];

if urls[4]!='' : path += '?' + urls[4]

dl.request(method="GET", url=path, headers=headers); rs = dl.getresponse()

if rs.status==200 :

R = parseData(rs.read(),url);

else :

print "3 seconds, try again ..."; time.sleep(3)

dl.request(method="GET", url=path, headers=headers); rs = dl.getresponse()

if rs.status==200 :

R = parseData(rs.read(),url);

else :

print "3 seconds, try again ..."; time.sleep(3)

dl.request(method="GET", url=path, headers=headers); rs = dl.getresponse()

if rs.status==200 :

R = parseData(rs.read(),url);

else :

print "Continue to collect ..."

R = 3

# 更新记录

updateOneUrl(url,R)

# 返回结果

return R

# 处理采集到的数据

def parseData(html,url):

global dl,conn; R = 2;

c = conn.cursor()

# 格式化html代码

format = formatURL(clearBlank(html),url)

# 取出所有的连接

urls = re.findall(r'''(]*?href="([^"]+)"[^>]*?>)|(]*?href='([^']+)'[^>]*?>)''',format,re.I)

if urls != None :

i = 0

# 循环所有的连接

for regs in urls :

# 得到一个单一的url

sUrl = en2chr(regs[1].strip())

# 判断url是否符合规则，符合，则插入数据库

if re.search('http(.*?)/(dir|question)/index(.*?)',sUrl,re.I) != None :

if re.search('http(.*?)/dir/index(.*?)',sUrl,re.I) != None:

if sUrl.find('link=list') == -1 and sUrl.find('link=over') == -1 :

sUrl+= '&link=over'

else:

sUrl = sUrl.replace('link=list','link=over')

if sUrl[-11:]=='link=mailto' : continue

try :

c.execute('INSERT INTO [collect]([curl])VALUES(?);',(sUrl,))

i = i + 1

except sqlite.IntegrityError :

pass

if i>0 : print "Message: %d get a new URL." % (i,)

# 截取数据

if re.search('http(.*)/question/index(.*)',url,re.I) != None :

sortfoot = 0

# 自动创建分类和分类关系

guide  = sect(format,'

1. ','

','((.*?)Home(.*?))')

aGuide = re.findall(']*href="[^"]*"[^>]*>(.*?)',guide,re.I)

if aGuide != None :

sortname = ""

for sortname in aGuide :

sortname = sortname.strip()

sortpath = en2path(sortname)

# 查询分类是否存在

c.execute('SELECT [sortid],[sortname] FROM [sorts] WHERE [sortpath]=? LIMIT 0,1;',(sortpath,))

row = c.fetchone();

# 分类不存在，添加分类

if row==None :

c.execute('INSERT INTO [sorts]([sortname],[sortpath],[sortfoot])VALUES(?,?,?);',(sortname,sortpath,sortfoot))

sortfoot = c.lastrowid

else:

sortfoot = row[0]

# 标题

title = sect(format,'

','

')

# 最佳答案

BestAnswer = sect(format,'(

Best Answer(.*?)

(.*?)

)')

# 最佳答案不存在，则不采集

if BestAnswer != None :

# 文章路径

path = en2path(sortname + '-' + title.strip())

# 问题

adddata = sect(format,'

')

content = sect(format,'(

(.*?)

)')

if adddata != None : content += '
' + adddata

# 其他回答

OtherAnswer = ''

for regs in re.findall('