python3 urlretrieve

import urllib.request
url = 'http://i3.hunantv.com/p1/20150906/1637244570C.jpg'
file_name = 'F:\\img\\ym.jpg'
...
# Download the file from `url` and save it locally under `file_name`:
urllib.request.urlretrieve(url, file_name)

python md5 加密

import hashlib
import timem = hashlib.md5()m.update(str(time.time()).encode())md5value=m.hexdigest()print(md5value)

python base64 加密，encodeURIComponent

import time
from urllib import parsetm = time.time()print(str(tm))tm = str(tm)[0:10]print(tm)import base64print(base64.b64encode(tm.encode()).decode())b64 = base64.b64encode(tm.encode()).decode()print(parse.quote(b64)) # encodeURIComponent'''
MTUwMjQyODkwMw==
MTUwMjQyNTE1Mw==
'''

草榴 Spider

config

PROXIES = { "http": "http://xxx:xxxx", "https": "http://xxx:xxxx", }MONGO_URL = '127.0.0.1'
MONGO_DB = 'caoliu'CLSQ = 'http://xxx.com/' # 地址就不留了
CLSQ_DOWNLOAD = 'http://www.xxx.com/download.php?'BT_PATH = 'F:\\1024bt\\{}\\'

viidii

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import Byfrom pyquery import PyQuery as pqdef get_b64(art_hash):browser = webdriver.PhantomJS(service_args=['--load-images=false', '--disk-cache=true'])browser.set_window_size(1400, 900)wait = WebDriverWait(browser, 10)browser.get('http://www.viidii.info/?http://www______rmdown______com/link______php?hash={}'.format(art_hash))wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'input[type=hidden]')))html = browser.page_sourcedoc = pq(html)b64 = doc.find('input[type=hidden]').attr('value')return b64

spider

import requests
import pymongo
import os
import re
# import base64
import time
import viidiifrom pyquery import PyQuery as pq
from Config import *from urllib import parse# db
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]
global table# 时间
import datetime
today = datetime.date.today()
one_day = datetime.timedelta(1)
yesterday = today - one_day
before_day = yesterday - one_dayyesterday_str = yesterday.strftime('%Y-%m-%d')
before_day_str = before_day.strftime('%Y-%m-%d')# 计数器
global countdef insert_to_mongo(info):global countglobal tablequery_info = {'art_name': info['art_name']}if table.find(query_info).limit(1).count() == 0:table.insert(info)count += 1print('成功插入第 ', count, ' 数据:', info)else:print('重复:', info)'''1. 插入mongo 之前，info 中被加入一条_id字段2. python 传递参数对于不可变对象作为函数参数，相当于C系语言的值传递；对于可变对象作为函数参数，相当于C系语言的引用传递。3. 所以如果新插入的info 如果不flush，mongo 将会报错'''def art_bt_hash(url):'''得到hash 码:param url: :return: '''doc = requests.get(url=url, proxies=PROXIES).content.decode('gbk')hash = re.search(r'rmdown\.com/link\.php\?hash=(.*?)<', doc, re.S).group(1) # 正则直接匹配return hashdef art_item(tr):'''得到一条（一件艺术品），获取这个art 的hash 码，对info 进行封装:param tr: :return: '''info = {}info['art_name'] = tr.find('h3').text()hash_url = tr.find('.tal > h3 > a').attr('href')art_url = '{}{}'.format(CLSQ, hash_url)info['art_url'] = art_urltry:info['art_hash'] = art_bt_hash(art_url)except AttributeError as e:print(e.args)returnexcept UnicodeDecodeError as e:print(e.args)returninfo['art_time'] = yesterday_strinfo['art_flag'] = '0'insert_to_mongo(info=info)def next_tags(**kwargs):'''翻页:param kwargs: :return: '''base = kwargs['base']page_num = kwargs['page_num']url = '{}&page={}'.format(base, page_num)print(url)try:doc = requests.get(url=url, proxies=PROXIES).contentexcept requests.exceptions.ContentDecodingError as e:print(e.args)time.sleep(2)next_tags(base=base, page_num=page_num)returnhtml = pq(doc)trs = html.find('#ajaxtable > tbody:nth-child(2) > tr').items()for tr in trs:art_time = tr.find('div[class=f10]').text()if art_time == before_day_str:returnif art_time == yesterday_str:art_item(tr)next_tags(base=base, page_num=page_num+1)# 得到 hash 码，然后放入mongodb
def art_tags(**kwargs):url = kwargs['url']print(url)doc = requests.get(url=url, proxies=PROXIES).contenthtml = pq(doc)trs = html.find('#ajaxtable > tbody:nth-child(2) > tr').items()flag = Falsefor tr in trs:if flag:art_time = tr.find('div[class=f10]').text()if art_time == before_day_str:returnif art_time == yesterday_str:art_item(tr)if tr.text() == '普通主題':flag = Truenext_tags(base=url, page_num=2)def downloader(**kwargs):'''下载器r = requests.get(url).contentwith open(file=path, mode='wb') as f:f.write(r):param kwargs: :return: '''url = kwargs['url']hash = kwargs['hash']r = requests.get(url).contentprint(url, r)path = '{}{}.torrent'.format(BT_PATH.format(yesterday_str), hash)try:with open(file=path, mode='wb') as f:f.write(r)except FileNotFoundError as e:print(e.args)return Falseprint('bt -> ', path)return True
# bt 下载器，从mongodb 中得到hash，下载bt
def art_bt_download(**kwargs):global tablequery_info = kwargs['query_info']for item in table.find(query_info):art_hash = item['art_hash']# stamp_base64 = parse.quote(base64.b64encode(str(time.time())[0:10].encode()).decode())stamp_base64 = parse.quote(viidii.get_b64(art_hash=art_hash))url = '{}ref={}&reff={}&submit=download'.format(CLSQ_DOWNLOAD, art_hash, stamp_base64)if downloader(url=url, hash=art_hash):update_data = {'$set' : {'art_flag' : '1'}}table.update(spec=item, document=update_data, upsert=False)else:table.remove(item)print('删除一条数据...')if __name__ == '__main__':global count# 创建文件夹if not os.path.exists(BT_PATH.format(yesterday_str)):os.makedirs(BT_PATH.format(yesterday_str))# 类别listurl_dict = {'2' : '亞洲無碼原創區', '15' : '亞洲有碼原創區', '5' : '動漫原創區'}# 遍历for type, name in url_dict.items():count = 0global tabletable = db[name]# art_tags(url='{}thread0806.php?fid={}'.format(CLSQ, type))print('启动下载器...')# art_bt_download(query_info={'art_flag': '0'})art_bt_download(query_info={'art_time' : '2017-08-10'})

python - 1024 Spider相关推荐

python的spider程序下载_PHPspider爬虫10分钟快速教程（内附python教程分享）
说到做爬虫,大家都可能第一时间想到的是python,其实php也是可以用来写爬虫程序的.php一贯简洁.易用,亲测使用PHPspider框架10分钟就能写出一个简单的爬虫程序. 一.PHP环境安装和 ...
python中spider的用法_python网络爬虫 CrawlSpider使用详解
CrawlSpider 作用:用于进行全站数据爬取 CrawlSpider就是Spider的一个子类如何新建一个基于CrawlSpider的爬虫文件 scrapy genspider -t craw ...
python中spider的用法_Spider-PyQuery基本用法(示例代码)
pip install pyquery 2.引用方法 from pyquery import PyQuery as pq 3.简介 pyquery 是类型jquery 的一个专供python使用的ht ...
python与spider的区别_python – Scraw spider与Scraped items之间的区别
我正在编写一个Scrapy CrawlSpider,它在第一页上读取AD列表,获取一些信息,如列表和AD网址,然后向每个AD网址发出请求以获取其详细信息. 它在测试环境中工作和分页显然很好,但今天试图 ...
Python爬虫实战（4）：抓取淘宝MM照片
Python爬虫入门(1):综述 Python爬虫入门(2):爬虫基础了解 Python爬虫入门(3):Urllib库的基本使用 Python爬虫入门(4):Urllib库的高级用法 Python爬虫 ...
【python爬虫】学习笔记1-爬取某网站妹子图片
最近刚刚学习爬虫,看了视频之后准备自己写一个爬虫练习,爬妹子图这次准备爬取的网站是:http://www.umei.cc/p/gaoqing/xiuren_VIP/ 接下来先说一下我的思路: 首先我 ...
Java调用Python下载网页
本篇参考:http://tonl.iteye.com/blog/1918245 python版本:2.7 64bit window版本: 下载python:http://www.python.org/ ...
Python爬虫实战：抓取淘宝MM照片
本篇目标 1.抓取淘宝MM的姓名,头像,年龄 2.抓取每一个MM的资料简介以及写真图片 3.把每一个MM的写真图片按照文件夹保存到本地 4.熟悉文件保存的过程 1.URL的格式在这里我们用到的URL ...
python脚本一键抓考试资料网答案
简介:本文介绍使用python抓取考试资料网的答案,将答案写到一个html里面,支持多url输入.解决考试资料网上面答案不能被复制的问题. 使用方法找到你需要的题目,如http://www.ppka ...
python利用selenium/requests/bs4/xpath爬虫知网8516本学术期刊具体概要
作业描述基于requests爬虫模块库, 把所有学术期刊的简介信息爬取下来导航页网址: https://navi.cnki.net/knavi/Journal.html 要求: 爬取所有学术期刊的 ...

python - 1024 Spider

python3 urlretrieve

python md5 加密

python base64 加密，encodeURIComponent

草榴 Spider

config

viidii

spider

python - 1024 Spider相关推荐

最新文章

热门文章