KeysdeMacBook:Desktop keys$ scrapy startproject Novel
New Scrapy project 'Novel', using template directory '/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/scrapy/templates/project', created in:/Users/keys/Desktop/Novel
You can start your first spider with:cd Novelscrapy genspider example example.com

KeysdeMacBook:Desktop keys$ cd Novel/
KeysdeMacBook:MyCrawl keys$ scrapy genspider BiJi www.mossiella.com/html/255.html

# -*- coding: utf-8 -*-import scrapyclass NovelItem(scrapy.Item):url = scrapy.Field()title = scrapy.Field()content = scrapy.Field()

# -*- coding: utf-8 -*-
import scrapy
from Novel.items import NovelItemclass BijiSpider(scrapy.Spider):name = 'BiJi'allowed_domains = ['www.mossiella.com']start_urls = ['http://www.mossiella.com/html/255.html/']def parse(self, response):url = response.urltitle = response.xpath('//h1/text()').extract_first('')content = ','.join(response.xpath('//div[@class="zwcent"]/p/text()').extract())myitem = NovelItem()myitem['url'] = urlmyitem['title'] = titlemyitem['content'] = contentyield myitemnext_url = response.xpath('//a[@rel="next"]')if next_url:url = next_url.css('::attr(href)').extract_first('')yield scrapy.Request(url=url, callback=self.parse)

import pymysqlclass MysqlPipeline(object):# 采用同步的机制写入mysqldef __init__(self):self.conn = pymysql.connect('127.0.0.1','root','rootkeys','Article',charset="utf8",use_unicode=True)self.cursor = self.conn.cursor()def process_item(self, item, spider):insert_sql = """insert into BiJi(url, title, content)VALUES (%s, %s, %s)"""# 使用VALUES实现传值self.cursor.execute(insert_sql,(item["url"],item["title"],item["content"]))self.conn.commit()

import os
import sys
from scrapy.cmdline import executesys.path.append(os.path.dirname(os.path.abspath(__file__)))run_spider = 'BiJi'if __name__ == '__main__':print('Running Spider of ' + run_spider)execute(['scrapy', 'crawl', run_spider])