代码语言：javascript
复制# coding:utf-8
import requests
from lxml import etree
import json
class Tieba(object):
def __init__(self, name):
self.url = 'https://tieba.baidu.com/f?kw={}'.format(name)
self.heardes = {
"User-Agent": "Mozilla/4.0 (compatible; MSIE 5.00; Windows 98)"
}
def get_data(self, url):
response = requests.get(url, headers=self.heardes)
with open("temp.html", "wb") as f:
f.write(response.content)
return response.content
def parse_data(self, data):
# 创建element对象
# data = data.decode().replace("<!--", "")
html = etree.HTML(data)
el_list = html.xpath('//li[@class=" j_thread_list clearfix thread_item_box"]/div/div[2]/div[1]/div[1]/a')
print(len(el_list))
data_list = []
for el in el_list:
temp = {}
temp['title'] = el.xpath("./text()")[0]
temp['link'] = 'https://tieba.baidu.com' el.xpath("./@href")[0]
data_list.append(temp)
# 获取下一页url
try:
next_url = 'https:' html.xpath('//a[contains(text(),"下一页>")]/@href')[0]
except:
next_url = None
return data_list, next_url
def save_data(self, data_list):
for data in data_list:
with open("temp.json", "a ", encoding='utf-8') as f:
data_temp = json.dumps(data, ensure_ascii=False)
print(data_temp)
f.write(data_temp)
# f.write(data)
def run(self):
# url
# hearders
next_url = self.url
while True:
# 发送请求,获取响应
data = self.get_data(self.url)
# 从相应中提取数据(数据和翻页用的url)
data_list, next_url = self.parse_data(data)
self.save_data(data_list)
print(next_url)
# 判断是否终结
if next_url == None:
break
if __name__ == '__main__':
tieba = Tieba("华东理工大学吧")
tieba.run()
