From 8dbc104feb3d5b3d12262ee32f092830db0e674a Mon Sep 17 00:00:00 2001 From: ygcaicn Date: Mon, 9 Dec 2019 11:48:22 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=85=95=E8=AF=BE?= =?UTF-8?q?=E8=A7=86=E9=A2=91=E4=B8=8B=E8=BD=BD=EF=BC=8C=E8=8E=B7=E5=8F=96?= =?UTF-8?q?m3u8=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooc/icourse163.py | 49 +++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/mooc/icourse163.py b/mooc/icourse163.py index 27c8514..2b1a084 100644 --- a/mooc/icourse163.py +++ b/mooc/icourse163.py @@ -3,6 +3,7 @@ import time from .utils import * +import json CANDY = Crawler() CONFIG = {} @@ -11,21 +12,21 @@ def get_summary(url): """从课程主页面获取信息""" + res = CANDY.get(url) + cookie_csrf = res.cookies.get('NTESSTUDYSI') + res_t = res.text - res = CANDY.get(url).text - - term_id = re.search(r'termId : "(\d+)"', res).group(1) - names = re.findall(r'name:"(.+)"', res) + term_id = re.search(r'termId : "(\d+)"', res_t).group(1) + names = re.findall(r'name:"(.+)"', res_t) dir_name = course_dir(*names[:2]) print(dir_name) - return term_id, dir_name + return term_id, dir_name, cookie_csrf def parse_resource(resource): """解析资源地址和下载资源""" - post_data = {'callCount': '1', 'scriptSessionId': '${scriptSessionId}190', 'httpSessionId': '5531d06316b34b9486a6891710115ebc', 'c0-scriptName': 'CourseBean', 'c0-methodName': 'getLessonUnitLearnVo', 'c0-id': '0', 'c0-param0': 'number:' + resource.meta[0], @@ -34,21 +35,28 @@ def parse_resource(resource): res = CANDY.post('https://www.icourse163.org/dwr/call/plaincall/CourseBean.getLessonUnitLearnVo.dwr', data=post_data).text + file_name = resource.file_name if resource.type == 'Video': - resolutions = ['Shd', 'Hd', 'Sd'] - for sp in resolutions[CONFIG['resolution']:]: - # TODO: 增加视频格式选择 - # video_info = re.search(r'%sUrl="(?P.*?(?P\.((m3u8)|(mp4)|(flv))).*?)"' % sp, res) - video_info = re.search(r'(?Pmp4)%sUrl="(?P.*?\.(?P=ext).*?)"' % sp, res) - if video_info: - url, ext = video_info.group('url', 'ext') - ext = '.' + ext - break - res_print(file_name + ext) - FILES['renamer'].write(re.search(r'(\w+\.((m3u8)|(mp4)|(flv)))', url).group(1), file_name, ext) - FILES['video'].write_string(url) - resource.ext = ext + post_data = {'bizId': resource.meta[2], + 'bizType': '1', + 'contentType': '1'} + url = 'https://www.icourse163.org/web/j/resourceRpcBean.getResourceToken.rpc?csrfKey={}'.format(CONFIG['_cookie_csrf']) + res = CANDY.post(url,data=post_data).text + res_json = json.loads(res) + if res_json.get('code', -1) != 0: + print('get signature error.') + return + sig = res_json['result']['videoSignDto']['signature'] + + url = 'https://vod.study.163.com/eds/api/v1/vod/video?videoId={}&signature={}&clientType=1'.format(resource.meta[0], sig) + res = CANDY.get(url).text + res_json = json.loads(res) + url = res_json['result']['videos'][-1]['videoUrl'] + name = res_json['result']['name'] + FILES['video'].write_string('{}\t{}'.format(name,url)) + res_print('{}\t{}'.format(name, url)) + if not CONFIG['sub']: return @@ -165,7 +173,8 @@ def start(url, config): global WORK_DIR CONFIG.update(config) - term_id, dir_name = get_summary(url) + term_id, dir_name, cookie_csrf = get_summary(url) + CONFIG['_cookie_csrf'] = cookie_csrf WORK_DIR = WorkingDir(CONFIG['dir'], dir_name) WORK_DIR.change('Videos') From e7553652cfdd6d270d665fa0712ea4df7d5a6d3d Mon Sep 17 00:00:00 2001 From: ygcaicn Date: Mon, 9 Dec 2019 11:48:43 +0800 Subject: [PATCH 2/4] =?UTF-8?q?m3u8=E4=B8=8B=E8=BD=BD=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- download_m3u8.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 download_m3u8.py diff --git a/download_m3u8.py b/download_m3u8.py new file mode 100644 index 0000000..3829694 --- /dev/null +++ b/download_m3u8.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import os +import sys +import argparse + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Download m3u8 to mp4') + parser.add_argument('-l', default=r'', help='下载目录') + parser.add_argument('-d', default=r'', help='下载目录') + args = parser.parse_args() + video_list = os.path.abspath(args.l) + + if args.d == '': + video_dir = os.path.dirname(video_list) + else: + video_dir = os.path.abspath(args.d) + + with open(video_list) as f: + lines = f.readlines() + lines = [_.strip() for _ in lines if _ != '\n'] + for line in lines: + name,url = line.split('\t') + print(name,url) + os.system('ffmpeg -i "{}" -c copy -bsf:a aac_adtstoasc "{}"'\ + .format(url,os.path.join(video_dir,name))) \ No newline at end of file From 34dc3d0d2386607aee304df05d69d3524df671a3 Mon Sep 17 00:00:00 2001 From: ygcaicn Date: Mon, 9 Dec 2019 11:54:48 +0800 Subject: [PATCH 3/4] =?UTF-8?q?m3u8=E4=B8=8B=E8=BD=BD=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E4=BD=BF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 50478b4..fc5b8c8 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,13 @@ 详细信息和用法请见 [https://mooc.xoy.io/](https://mooc.xoy.io/)。 +### m3u8下载脚本使用 +依赖系统ffmpeg, 需要先安装ffmpeg， + +```sh +./download_m3u8.py -l 学术规范与论文写作/Videos/Videos.txt +``` + ### 声明 仅限个人学习和研究使用，切勿用于其他用途。强烈建议到 MOOC 网站进行学习，本程序只是提供一个备选方案。 From a9993ba486252b7addf48236f38cfcc1876861cd Mon Sep 17 00:00:00 2001 From: ygcaicn Date: Mon, 9 Dec 2019 11:56:57 +0800 Subject: [PATCH 4/4] =?UTF-8?q?m3u8=E4=B8=8B=E8=BD=BD=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- download_m3u8.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fc5b8c8..207b79c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ 详细信息和用法请见 [https://mooc.xoy.io/](https://mooc.xoy.io/)。 ### m3u8下载脚本使用 -依赖系统ffmpeg, 需要先安装ffmpeg， +依赖系统ffmpeg, 需要先安装ffmpeg， 在Ubuntu下已测试。 ```sh ./download_m3u8.py -l 学术规范与论文写作/Videos/Videos.txt diff --git a/download_m3u8.py b/download_m3u8.py index 3829694..e8ef15c 100644 --- a/download_m3u8.py +++ b/download_m3u8.py @@ -5,7 +5,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='Download m3u8 to mp4') - parser.add_argument('-l', default=r'', help='下载目录') + parser.add_argument('-l', default=r'', help='视频列表') parser.add_argument('-d', default=r'', help='下载目录') args = parser.parse_args() video_list = os.path.abspath(args.l)