import time
import requests
from bs4 import BeautifulSoup
import os
import re
# import dill
import PySimpleGUI as sga = time.time()
# dill.load_session('file_name.pkl')
# dill.dump_session('file_name.pkl')#定义一个空集合用来保存更新的文件
update_slides = []
#获取课程url信息def getHTML(url):try:r = sessions.get(url, timeout=60, headers=hd)r.raise_for_status()  # 容错机制，若请求访问失败则返回的不是200，则返回字符串空r.encoding = r.apparent_encoding  # 设置编码方式，用解析返回网页源码得出的编码方式代替  UTF-8return r.textexcept:return ''#定义解析HTML函数
def soupanyl(course_url):text = getHTML(course_url)  # 获得课程主页面storage，减少重复运算soup = BeautifulSoup(text, 'html.parser')return soup#定义文件保存函数
def save_files(file_url, local_path, course_title, section_name, first_folder_name, file_name):global update_slidesfile_path = local_path + '/' + course_title + '/' + section_name + '/' + first_folder_name + '/' + file_namestore_path = file_path.replace(file_path.split('/')[-1], '')new_file = course_title + '/' + section_name + '/' + first_folder_name + '/' + file_nameif not os.path.exists(store_path):os.makedirs(store_path)if not os.path.exists(file_path):try:print('新上了一个文件正在准备下载' + new_file)start = time.time()response = sessions.get(file_url,stream=True)  # stream参数设置成True时，它不会立即开始下载，当你使用iter_content或iter_lines遍历内容或访问内容属性时才开始下载chunk_size = 1024  # 每次块大小为1024content_size = int(response.headers['content-length'])  # 返回的response的headers中获取文件大小信息size_data = str(round(float(content_size / chunk_size / 1024), 4))print("    文件大小：" + size_data + "[MB]")print('  ', end='')with open(file_path, 'wb') as file:count = 0for data in response.iter_content(chunk_size=chunk_size):  # 每次只获取一个chunk_size大小file.write(data)  # 每次只写入data大小count += 1024proced = int(round(100 * count / content_size))print(('r    进度: {}%' + '█' * round(proced / 2) + '>' * (50 - round(proced / 2))).format(proced),end='')  # 向终端输出#  [ r 加上文本 加 end = ''是单行刷新）end = time.time()print("n    该文件下载成功，总耗时:" + str(end - start) + "秒", end='n')print('', end='n')update_slides.append(new_file)except:print('    网络或其他原因，该文件下载失败。download failed')else:print('    该路径下文件已经存在:' + new_file)#从HTML解析出文件名
def seg(url):before = re.findall(r'/0/.+?forcedownload', url)[0][3:-14].replace('%20', ' ')return before#一部分文件名存在课件下载url链接中，但是在重新定向的文件中，定义两个函数重新定向获取url里的文件名
def redirect_url_name(url):# 获取pdf连接后名字的爬取redirect = sessions.get(url, timeout=60, headers=hd)# print(sessions.status_code)  # 打印响应的状态码return redirect.url  # 打印重定向后的网址def get_redi_name(url):redi_url = redirect_url_name(url)name = re.findall(r'/content/./.+', redi_url)[0][10:].replace('%20', ' ')return name#解析的文件夹名中删除那些建立文件夹不合法的特殊字符
def ex_odd(txt):for ch in "/:*?''<> |":txt = txt.replace(ch, " ")return txt#获得课程标题
def getitle(url):try:soup = soupanyl(url)title = soup.h1.stringif title[0] == ' ':title = title[1:]return titleelse:title = titlereturn titleexcept:return 'unknow'#定义下载某一课程的所有课件
def get_course_all_slides(course_url, local_path):soup = soupanyl(course_url)materials = soup('li', id=re.compile('section'))  # get section htmlcourse_title = getitle(course_url)for section in materials:try:infos = section('a')section_name = section.span.stringsection_name = ex_odd(section_name)for info in infos:try:url1 = info.get('href')if 'resource/' in url1:file1_name = get_redi_name(url1)first_folder_name = ''save_files(url1, local_path, course_title, section_name, first_folder_name, file1_name)elif 'download' in url1:first_folder_name = ''file2_name = seg(url1)save_files(url1, local_path, course_title, section_name, first_folder_name, file2_name)elif '/folder/' in url1:file_or_folder_name = info.span.contents[0]soup2 = soupanyl(url1)materials2 = soup2('div', role='main')msgs = materials2[0]('a')for msg in msgs:try:url2 = msg.get('href')file3_name = seg(url2)save_files(url2, local_path, course_title, section_name, file_or_folder_name,file3_name)except:continueelse:continueexcept:continueexcept:continue
#开始访问并登录网站，session用来保存网站的cookie信息，换句话说就是记中账号密码，如果不是这样，后边的url解析都是未登录状态。
sessions = requests.session()
hd = {'user-agent': 'chorme/10'}        #定义浏览器标头，防止爬虫禁止的条令
def login(UserName,PassWord):print('    准备进入Moodle....(这个网站会卡，正常的，别担心，等等就好，如果一直进不去的话的话打开浏览器看看卡不卡，要是也卡的话就不是我的锅了，我是robot，和你手动下载一样的，不用你点点点了),Preparing for entering Moodle.......')loginurl = 'https://sdc-moodle.samf.aau.dk/login/index.php'data = {'username': UserName, 'password':PassWord}request = sessions.post(url=loginurl, data=data, headers=hd)new_lo = request.urlreturn new_lo == loginurl #返回是否登录成功的信息，True代表未成功，False是成功的，如果登录成功url会改变。##下面是url设计 PysimpleGUI是一种很简单的GUI库，可以在很短的周期学会，很适合没有时间学习第三方GUI库的初学者甚至熟练的程序猿。sg.ChangeLookAndFeel('TanBlue')     #GUI主题# Design pattern 1 - First window does not remain active
#设计GUI布局
layout = [[sg.Text('Moodle-Slides-Downloader for SDC students',font=("Reds", 25),size=(45, 1))],[sg.Text('账号（UserName）:',font=(15)),sg.Input(key = '_USER_',size=(30, 1))],[sg.Text('密码（Password）:',font=(15)),sg.Input(password_char='*', key='_PWD_',size=(30, 1)),],[sg.Text('路径（Filepath）:',font=(15)),sg.InputText('Select your curricula storage path',key = '_PATH_'), sg.FolderBrowse()],[sg.Checkbox('',default= True,key = 'cs1'),sg.Text('课程1(Course1):', font=(15)), sg.InputText('Please input your course URL', size=(60, 1),key = 'course1')],[sg.Checkbox('',default = True,key = 'cs2'),sg.Text('课程2(Course2):', font=(15)), sg.InputText('Please input your course URL', size=(60, 1),key = 'course2')],[sg.Checkbox('',default = True,key = 'cs3'),sg.Text('课程3(Course3):', font=(15)), sg.InputText('Please input your course URL', size=(60, 1),key = 'course3')],[sg.Checkbox('',default = True,key = 'cs4'),sg.Text('课程4(Course4):', font=(15)), sg.InputText('Please input your course URL', size=(60, 1),key = 'course4')],[sg.Btn('SaveSettings'),sg.Button('LoadSettings'),sg.Btn('Submit')],
]#设计GUI页眉
win = sg.Window('中丹学院(Sino-Danish Center)     @Jinglu Han     version1.0', layout)
#设置主事件循环，保持窗口一致打开，供用户输入信息while True:ev, vals = win.Read()filename = 'Moodle_downloader_save.sav'if ev == 'SaveSettings':try:win.SaveToDisk(filename)        #PySimpleGUI里面的保存设置按钮，找了很久的API，供大家参考sg.popup('sucessfully saved')except:sg.popup('Please select your storage path')elif ev == 'LoadSettings':win.LoadFromDisk(filename)# load(form)if ev is None:breakif ev == 'Submit':judge = login(vals['_USER_'],vals['_PWD_'])if judge:sg.popup('login failed,please check your account or password')else:sg.popup('sucessfully login in,start downloading')print('    成功进入Moodle，sucessfully login in Moodle')for  i in range(4):try:T = vals['cs' + str(i+1)]if T:course_url = vals['course' + str(i+1)]local_path = vals['_PATH_']get_course_all_slides(course_url, local_path)else:continueexcept:continueb = time.time()print('程序运行时间：{:.2f}秒'.format(b - a))print('程序运行时间：{:.2f}分'.format((b - a) / 60))print('更新以下文件：(Below files have been downloaded)')for update in update_slides:print(update + '已下载(has been downloaded)')