2018/05/29 修改抓取编码gb2312改gb18030

项目需要行政区域三级联动，刚好写个爬虫练练手。

Laravel 框架，安装的两个库

 composer require guzzlehttp/guzzlecomposer require symfony/dom-crawler

创建表

DROP TABLE IF EXISTS `area`;
CREATE TABLE `area` (`id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,`name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,`parent_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;DROP TABLE IF EXISTS `crawler`;
CREATE TABLE `crawler` (`id` int(11) NOT NULL AUTO_INCREMENT,`status` int(11) DEFAULT '0',`data` text COLLATE utf8mb4_unicode_ci,PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

创建对应Model

App\Model\Area.php

<?phpnamespace App\Model;use Illuminate\Database\Eloquent\Model;class Area extends Model
{public $timestamps = false;protected $table = 'area';protected $keyType = 'string';protected $fillable = ['id', 'name', 'parent_id',];
}

App\Model\Crawler.php

<?phpnamespace App\Model;use Illuminate\Database\Eloquent\Model;class Crawler extends Model
{public $timestamps = false;protected $table = 'crawler';protected $fillable = ['id', 'status', 'data'];
}

app/Console/Kernel.php 添加

    protected $commands = ['App\Console\Commands\CityCrawler',  ];

新建 App\Console\Commands\CityCrawler.php

<?php
namespace App\Console\Commands;use Illuminate\Console\Command;
use Symfony\Component\DomCrawler\Crawler;use App\Model\Area;
use App\Model\Crawler as CrawlerTask;use GuzzleHttp\Psr7;
use GuzzleHttp\Exception\RequestException;// 流程：
//     1. func top 抓取行政区域省级, 每个省链接生成一次抓取任务，保存到任务表crawler。
//     2. 循环抓取
//             1). 读取一条任务 select * from crawler where status = 0 limit 1; update crawler set status = 1 where id = 本次任务id;
//             2). 根据任务类型调用抓取方法 如 镇抓取：crawler_towntr 区抓取crawler_districts 城市抓取crawler_citys
//                 抓取方法中保存抓取到的行政区域数据到area表，并根据抓取行政区域下一级生成下一次抓取任务，存放任务表
class CityCrawler extends Command
{protected $signature = 'street:crawler';protected $description = 'Street Crawler';protected $start_url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/';protected $special_city = ['东莞市','中山市','嘉峪关市','三沙市','儋州市']; // 中国5个不设市辖区的地级市 public function handle(){//抓取省级行政区域$this->top(); while (true) {$task_model = $this->task();if(empty($task_model)){return $this->info("End");}$task= json_decode($task_model->data, true);// 打印日志$this->info(implode(',', array_map(function($item){return $item['id'] . ' ' . $item['name'];}, $task['data'])));$status = call_user_func(array($this, 'crawler_' . $task['crawler']), $task);if($status){$this->finish($task_model);}else{var_dump($task, 'error');return false;}$this->info("sleep 1");sleep(1);}}public function finish($task){$task->status = 2;$task->save();}public function task($status = 0){$task = CrawlerTask::where("status", $status)->first();$task->status = 1; // 进行中$task->save();return $task;}public function push($data){$task = new CrawlerTask;$task->data = json_encode($data);$task->save();}// 第一个页面public function top(){$url = $this->start_url;$html = $this->send_http($url);$crawler = new Crawler();$crawler->addHtmlContent($html, 'gb18030');$crawler->filter('.provincetr')->filter('td > a')->each(function(Crawler $node, $i) use($url) {$text = $node->text();$href = $node->attr('href');$id = str_replace('.html', '', $href);$task = ['crawler' => 'citys','remark' => '省','url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,'data' => [ [ 'name' => $text, 'id' => $id] ],'parent_id' => $id,];$this->push($task);Area::create(['id' => $id,'name' => $text,'parent_id' => 0]);$this->info($node->attr('href'));$this->info($text);});}public function crawler_towntr($task){$url = $task['url'];if(!strpos($url, '.html')){$this->info('为空的直辖市');return true;}$html = $this->send_http($url);$crawler = new Crawler();$crawler->addHtmlContent($html, 'gb18030');$crawler->filter('.towntr')->each(function(Crawler $node, $i) use ($task, $url) {$code_node = $node->filter('td')->eq(0)->filter('a');$name_node = $node->filter('td')->eq(1)->filter('a');Area::create(['id' => $code_node->text(),'name' => $name_node->text(),'parent_id' => $task['parent_id']]);$this->info($code_node->text() . '  ' . $name_node->text());});return true;}public function crawler_districts($task){$url = $task['url'];$html = $this->send_http($url);$crawler = new Crawler();$crawler->addHtmlContent($html, 'gb18030');$crawler->filter('.countytr')->each(function(Crawler $node, $i) use ($task, $url) {$code_node = $node->filter('td')->eq(0)->filter('a');$name_node = $node->filter('td')->eq(1)->filter('a');//没有子节点if($code_node->count() == 0){   $code_node = $node->filter('td')->eq(0);$name_node = $node->filter('td')->eq(1);}else{$href = $code_node->attr("href");$data = $task['data'];$data[] = ['name' => $name_node->text(), 'id' => $code_node->text()] ;$new_task = ['crawler' => 'towntr','remark' => '县 区','url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,'data' => $data,'parent_id' => $code_node->text(),];$this->push($new_task);                }$this->info($code_node->text() . '  ' . $name_node->text());Area::create(['id' => $code_node->text(),'name' => $name_node->text(),'parent_id' => $task['parent_id']]);});return true;}public function crawler_citys($task){$url = $task['url'];$html = $this->send_http($url);$crawler = new Crawler();$crawler->addHtmlContent($html, 'gb18030');$crawler->filter('.citytr')->each(function(Crawler $node, $i) use ($task, $url) {$code_node = $node->filter('td')->eq(0)->filter('a');$name_node = $node->filter('td')->eq(1)->filter('a');$href = $code_node->attr("href");$this->info($code_node->text() . '  ' . $name_node->text());Area::create(['id' => $code_node->text(),'name' => $name_node->text(),'parent_id' => $task['parent_id']]);$data = $task['data'];$data[] = ['name' => $name_node->text(), 'id' => $code_node->text()] ;if(in_array($name_node->text(), $this->special_city)){$new_task = ['crawler' => 'towntr','remark' => '特别的5个省地级市','url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,'data' => $data,'parent_id' => $code_node->text(),];                }else{$new_task = ['crawler' => 'districts','remark' => '城市','url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,'data' => $data,'parent_id' => $code_node->text(),];}$this->push($new_task);});return true;}public function info($string, $verbosity = null){$string = iconv( 'UTF-8', 'GB18030', $string); // cmd 中文gbk编码parent::line($string, 'info', $verbosity);}private function send_http($url){   $user_agent_list = ['Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.04','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.9 Safari/537.36','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 OPR/48.0.2685.52','Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0','Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27','Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',];$user_agent = $user_agent_list[(time() % 6)];$timeout = 5; // 秒$client = new \GuzzleHttp\Client(['headers' => ['User-Agent' => $user_agent], 'timeout' => $timeout]);try {$res  = $client->request('GET', $url);$html =  (string)$res->getBody();} catch (RequestException $e) {// 抓取中会有404状态返回，再重新请求一次。$this->info(Psr7\str($e->getRequest()));if ($e->hasResponse()) {$this->info(Psr7\str($e->getResponse()));}$this->info("send_http timeout retry");$this->info("sleep 2s");sleep(2);$res  = $client->request('GET', $url);$html =  (string)$res->getBody();              }return $html;}}

进目录运行

php artisan street:crawler

最后

------------------------------------------------------

数据有了全部写到一个json文件里，太大了1M多 :(

还是写成ajax从服务端读取三级联动数据。

area.js

// 1. 省加载 其他请选择
// 2. 省 change 触发加载市
// 3. 市触发加载区
// 4. 区加载触发街道
function area_init(param) {var area = this;area.area_not_filter = false;;area.prompt_html = '<option value="">－请选择－</option>';if (param.area_not_filter) {area.area_not_filter = param.area_not_filter;}if (param.province) {area.province_el = $(param.province);}if (param.city) {area.city_el = $(param.city);}if (param.district) {area.district_el = $(param.district);}if (param.street) {area.street_el = $(param.street);}area.load = function() {area.province_el.html(area.prompt_html);area.city_el.html(area.prompt_html);area.district_el.html(area.prompt_html);if (area.street_el) {area.street_el.html(area.prompt_html);}province_id = area.province_el.attr("data-value");city_id = area.city_el.attr("data-value");district_id = area.district_el.attr("data-value");if (area.street_el) {street_id = area.street_el.attr("data-value");}area.area_fill(0, 'province', area.province_el, province_id);province_id && area.area_fill(province_id, 'city', area.city_el, city_id);city_id && area.area_fill(city_id, 'district', area.district_el, district_id);if (area.street_el) {district_id && area.area_fill(district_id, 'street', area.street_el, street_id);}}area.bind = function() {area.province_el.change(function() {area.area_fill($(this).val(), 'city', area.city_el);area.city_el.html(area.prompt_html);area.district_el.html(area.prompt_html);if (area.street_el) {area.street_el.html(area.prompt_html);}});area.city_el.change(function() {area.area_fill($(this).val(), 'district', area.district_el);area.district_el.html(area.prompt_html);if (area.street_el) {area.street_el.html(area.prompt_html);}});if (area.street_el) {area.district_el.change(function() {area.area_fill($(this).val(), 'street', area.street_el);area.street_el.html(area.prompt_html);});}};area.area_fill = function(parent_id, level, el, active_id) {// value='' 不请求  ajaxif (parent_id === '') {return false;}area.get_area(parent_id, level, function(list) {var province = area.prompt_html;$.each(list, function(i, n) {province += '<option ' + (active_id == n.id ? ' selected ' : ' ') + ' value="' + n.id + '">' + n.name + '</option>';});$(el).html(province);});}area.get_area = function(parent_id, level, callback) {var $url = '/ajax_area/' + level + '/' + parent_id;$.ajax({url: $url,type: 'get',success: function(res) {callback(res);}});}area.load();area.bind();
}// 使用方法
// data-value 默认值
// 32 江苏省
// 320100000000 南京市
// 320102000000 玄武区
// <select name="province" data-value="32"></select>
// <select name="city" data-value="320100000000"></select>
// <select name="district" data-value="320102000000"></select>
// <select name="street" data-value=""></select>// var area = new area_init(
//         {
//             province: "select[name='province']",
//             city: "select[name='city']",
//             district: "select[name='district']",
//             street: "select[name='street']"
//         }
//     );// 或者// <select name="province" data-value="32"></select>
// <select name="city" data-value="320100000000"></select>// var area = new area_init(
//         {
//             province: "select[name='province']",
//             city: "select[name='city']"
//         }
//     );// 如果需要js动态修改
// $('select[name="province"]').attr('data-value', data.province);
// $('select[name="city"]').attr('data-value', data.city);
// $('select[name="district"]').attr('data-value', data.district);
// area.load();

成品长这样哈