Python爬取微信公眾號文章、點(diǎn)贊數
優(yōu)采云 發(fā)布時(shí)間: 2020-08-13 18:312、2 關(guān)注須要爬取的公眾號
2、3 設置Fiddler 并保存配置
微信客戶(hù)端懇求公眾號文章， fiddler抓取到的數據req_id 、 pass_ticket 、 appmsg_tojen
微信客戶(hù)端懇求公眾號文章， fiddler抓取到的數據cookie 和user-agent
最后恢復Fiddler的配置，會(huì )報錯的
END?。?！前面的獲取工作就結束了…
一、獲取cookie
# -*- coding: utf-8 -*-
import time
import json
from selenium import webdriver
import sys
sys.path.append('/path/to/your/module')
post = {}
driver = webdriver.Chrome(executable_path="chromedriver.exe")
driver.get('https://mp.weixin.qq.com')
time.sleep(2)
driver.find_element_by_name('account').clear()
driver.find_element_by_name('account').send_keys('訂閱號賬號')
driver.find_element_by_name('password').clear()
driver.find_element_by_name('password').send_keys('訂閱號密碼')
# 自動(dòng)輸入密碼后點(diǎn)擊記住密碼
time.sleep(5)
driver.find_element_by_xpath("./*//a[@class='btn_login']").click()
# 掃碼
time.sleep(20)
driver.get('https://mp.weixin.qq.com')
cookie_items = driver.get_cookies()
for cookie_item in cookie_items:
post[cookie_item['name']] = cookie_item['value']
cookie_str = json.dumps(post)
with open('cookie.txt', 'w+', encoding='utf-8') as f:
f.write(cookie_str)
print('cookie write ok ...')
二、獲取文章數據（文章列表寫(xiě)入數據庫）
<p># -*- coding: utf-8 -*-
import time, datetime
import json
import requests
import re
import random
import MySQLdb
#設置要爬取的公眾號列表
gzlist=['ckxxwx']
# 打開(kāi)數據庫連接
db = MySQLdb.connect("localhost", "root", "123456", "wechat_reptile_data", charset='utf8' )
# 數據的開(kāi)始日期 - 結束日期
start_data = '20190630';
end_data = '20190430'
# 使用cursor()方法獲取操作游標
cursor = db.cursor()
# 獲取閱讀數和點(diǎn)贊數
def getMoreInfo(link, query):
pass_ticket = "VllmJYTSgRotAAiQn17Tw1v35AduDOg%252BLCq%252B07qi4FKcStfL%252Fkc44G0LuIvr99HO"
if query == 'ckxxwx':
appmsg_token = "1016_%2F%2Bs3kaOp2TJJQ4EKMVfI0O8RhzRxMs3lLy54hhisceyyXmLHXf_x5xZPaT_pbAJgmwxL19F0XRMWtvYH"
phoneCookie = "rewardsn=; wxuin=811344139; devicetype=Windows10; version=62060833; lang=zh_CN; pass_ticket=VllmJYTSgRotAAiQn17Tw1v35AduDOg+LCq+07qi4FKcStfL/kc44G0LuIvr99HO; wap_sid2=CIvC8IIDElxlWlVuYlBacDF0TW9sUW16WmNIaDl0cVhxYzZnSHljWlB3TmxfdjlDWmItNVpXeURScG1RNEpuNzFUZFNSZWVZcjE5SHZST2tLZnBSdDUxLWhHRDNQX2dEQUFBfjCasIvpBTgNQAE=; wxtokenkey=777"
mid = link.split("&")[1].split("=")[1]
idx = link.split("&")[2].split("=")[1]
sn = link.split("&")[3].split("=")[1]
_biz = link.split("&")[0].split("_biz=")[1]
# 目標url
url = "http://mp.weixin.qq.com/mp/getappmsgext"
# 添加Cookie避免登陸操作，這里的"User-Agent"最好為手機瀏覽器的標識
headers = {
"Cookie": phoneCookie,
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 MicroMessenger/6.5.2.501 NetType/WIFI WindowsWechat QBCore/3.43.1021.400 QQBrowser/9.0.2524.400"
}
# 添加data，`req_id`、`pass_ticket`分別對應文章的信息，從fiddler復制即可。
data = {
"is_only_read": "1",
"is_temp_url": "0",
"appmsg_type": "9",
'reward_uin_count': '0'
}
params = {
"__biz": _biz,
"mid": mid,
"sn": sn,
"idx": idx,
"key": '777',
"pass_ticket": pass_ticket,
"appmsg_token": appmsg_token,
"uin": '777',
"wxtoken": "777"
}
# 使用post方法進(jìn)行提交
content = requests.post(url, headers=headers, data=data, params=params).json()
# 提取其中的閱讀數和點(diǎn)贊數
if 'appmsgstat' in content:
readNum = content["appmsgstat"]["read_num"]
likeNum = content["appmsgstat"]["like_num"]
else:
print('請求參數過(guò)期！')
# 歇10s，防止被封
time.sleep(10)
return readNum, likeNum
#爬取微信公眾號文章，并存在本地文本中
def get_content(query):
#query為要爬取的公眾號名稱(chēng)
#公眾號主頁(yè)
url = 'https://mp.weixin.qq.com'
#設置headers
header = {
"HOST": "mp.weixin.qq.com",
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0"
}
#讀取上一步獲取到的cookies
with open('cookie.txt', 'r', encoding='utf-8') as f:
cookie = f.read()
cookies = json.loads(cookie)
#登錄之后的微信公眾號首頁(yè)url變化為：https://mp.weixin.qq.com/cgi-bin/home?t=home/index&lang=zh_CN&token=1849751598，從這里獲取token信息
response = requests.get(url=url, cookies=cookies)
token = re.findall(r'token=(\d+)', str(response.url))[0]
time.sleep(2)
#搜索微信公眾號的接口地址
search_url = 'https://mp.weixin.qq.com/cgi-bin/searchbiz?'
#搜索微信公眾號接口需要傳入的參數，有三個(gè)變量：微信公眾號token、隨機數random、搜索的微信公眾號名字
query_id = {
'action': 'search_biz',
'token' : token,
'lang': 'zh_CN',
'f': 'json',
'ajax': '1',
'random': random.random(),
'query': query,
'begin': '0',
'count': '5'
}
#打開(kāi)搜索微信公眾號接口地址，需要傳入相關(guān)參數信息如：cookies、params、headers
search_response = requests.get(search_url, cookies=cookies, headers=header, params=query_id)
#取搜索結果中的第一個(gè)公眾號
lists = search_response.json().get('list')[0]
#獲取這個(gè)公眾號的fakeid，后面爬取公眾號文章需要此字段
fakeid = lists.get('fakeid')
#微信公眾號文章接口地址
appmsg_url = 'https://mp.weixin.qq.com/cgi-bin/appmsg?'
#搜索文章需要傳入幾個(gè)參數：登錄的公眾號token、要爬取文章的公眾號fakeid、隨機數random
query_id_data = {
'token': token,
'lang': 'zh_CN',
'f': 'json',
'ajax': '1',
'random': random.random(),
'action': 'list_ex',
'begin': '0',#不同頁(yè)，此參數變化，變化規則為每頁(yè)加5
'count': '5',
'query': '',
'fakeid': fakeid,
'type': '9'
}
#打開(kāi)搜索的微信公眾號文章列表頁(yè)
appmsg_response = requests.get(appmsg_url, cookies=cookies, headers=header, params=query_id_data)
#獲取文章總數
if appmsg_response.json().get('base_resp').get('ret') == 200013:
print('搜索公眾號文章操作頻繁,?。?！')
max_num = appmsg_response.json().get('app_msg_cnt')
#每頁(yè)至少有5條，獲取文章總的頁(yè)數，爬取時(shí)需要分頁(yè)爬
num = int(int(max_num) / 5)
#起始頁(yè)begin參數，往后每頁(yè)加5
begin = 0 # 根據內容自定義
while num + 1 > 0 :
query_id_data = {
'token': token,
'lang': 'zh_CN',
'f': 'json',
'ajax': '1',
'random': random.random(),
'action': 'list_ex',
'begin': '{}'.format(str(begin)),
'count': '5',
'query': '',
'fakeid': fakeid,
'type': '9'
}
print('正在翻頁(yè)：--------------',begin)
time.sleep(5)
#獲取每一頁(yè)文章的標題和鏈接地址，并寫(xiě)入本地文本中
query_fakeid_response = requests.get(appmsg_url, cookies=cookies, headers=header, params=query_id_data)
fakeid_list = query_fakeid_response.json().get('app_msg_list')
for item in fakeid_list:
content_link=item.get('link')
content_title=item.get('title')
update_time=item.get('update_time')
timeArray = time.localtime(update_time)
DataTime = time.strftime("%Y%m%d", timeArray)
print(DataTime)
Yearsmonth = time.strftime("%Y%m", timeArray)
if DataTime
