爬虫之爬取百度贴吧
直接示例代码：
import requests
# from lxml import html
# etree = html.etree
from lxml import etree
class Tieba(object):
def __init__(self, name):
self.url = f'https://tieba.baidu.com/f?kw={name}'
self.headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
# 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) ' # 低端浏览器没有被注释掉
}
def get_data(self, url):
response = requests.get(url, headers=self.headers)
# 把浏览器响应的内容保存到本地，以便查看响应的源码
# with open('tieba.html', 'wb') as f:
# f.write(response.content)
return response.content
def parse_data(self, data):
# 创建element对象
data = data.decode().replace("", "") # 高端浏览器会把一些内容给注释掉的
el_html = etree.HTML(data)
el_list = el_html.xpath('//*[@id="thread_list"]/li/div/div[2]/div[1]/div[1]/a') # 此处输出的是对象
print(len(el_list))
data_list = []
for el in el_list:
tmp = {}
tmp['title'] = el.xpath('./text()')[0] # 此处xpath取出的数据是列表，所以加上索引[0]
tmp['href'] = 'http://tieba.com' + el.xpath('./@href')[0] # 此处取出的索引是相对路径，所以前面还要拼接字符串
data_list.append(tmp)
print(data_list)
try:
# next_url = 'https' + el_html.xpath('//a[@class="next pagination-item "]/@href')
next_url = 'https:' + el_html.xpath('//a[contains(text(),"下一页")]/@href')[0]
except:
next_url = None
return data_list, next_url
def save_data(self, data_list):
for data in data_list:
print(data)
def run(self):
# url
# headers
next_url = self.url
while True:
# 发送请求，获取响应
data = self.get_data(next_url)
# 从响应中提取数据（数据和翻页用的url）
data_list, next_url = self.parse_data(data)
self.save_data(data_list)
# 判断是否终结
if next_url == None:
break
if __name__ == '__main__':
tieba = Tieba('上海大学')
tieba.run()
运行效果：
将在百度贴吧爬取的数据爬取下来并写进csv文件中。
示例代码：
import requests
# from lxml import html
# etree = html.etree
import csv
from lxml import etree
class Tieba(object):
def __init__(self, name):
self.url = f'https://tieba.baidu.com/f?kw={name}'
self.headers = {
# 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
'User-Agent': '/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
# 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) ' # 低端浏览器没有被注释掉
}
self.f = open('上海理工大学吧.csv', 'w', encoding='utf-8-sig', newline="")
self.csv_write = csv.writer(self.f)
def get_data(self, url):
response = requests.get(url, headers=self.headers)
# 把浏览器响应的内容保存到本地，以便查看响应的源码
# with open('tieba.html', 'wb') as f:
# f.write(response.content)
return response.content
def parse_data(self, data):
# 创建element对象
data = data.decode().replace("", "") # 高端浏览器会把一些内容给注释掉的
el_html = etree.HTML(data)
# el_list = el_html.xpath('//*[@id="thread_list"]/li/div/div[2]/div[1]/div[1]/a') # 此处输出的是对象
el_list = el_html.xpath('//*[@id="thread_list"]/li/div') # 此处输出的是对象
print(len(el_list))
# exit()
data_list = []
for el in el_list:
tmp = {}
tmp['title'] = el.xpath('./div[2]/div[1]/div[1]/a/text()')[0] # 此处xpath取出的数据是列表，所以加上索引[0]
tmp['href'] = 'http://tieba.com' + el.xpath('./div[2]/div[1]/div[1]/a/@href')[0] # 此处取出的索引是相对路径，所以前面还要拼接字符串
try:
tmp['author'] = el.xpath('./div[2]/div[1]/div[2]/span[1]/span[1]/a/text()')[0]
except:
tmp['author'] = el.xpath('./div[2]/div[1]/div[2]/span[1]/span[1]/a/text()')
try:
tmp['reviewer'] = el.xpath('./div[2]/div[2]/div[2]/span[1]/a/text()')[0]
except:
tmp['reviewer'] = el.xpath('./div[2]/div[2]/div[2]/span[1]/a/text()')
try:
tmp['last_comment_time'] = el.xpath('./div[2]/div[2]/div[2]/span[2]/text()')[0]
except:
tmp['last_comment_time'] = el.xpath('./div[2]/div[2]/div[2]/span[2]/text()')
try:
tmp['comment'] = el.xpath('./div[2]/div[2]/div[1]/div/text()')[0]
except:
tmp['comment'] = el.xpath('./div[2]/div[2]/div[1]/div/text()')
data_list.append(tmp)
# print(data_list)
# 获取csv他属性值
a = []
dict = data_list[0]
for headers in sorted(dict.keys()): # 把字典的键取出来
a.append(headers)
header = a # 把列名给提取出来，用列表形式呈现
print(a)
self.csv_write.writerow(['title', 'href', 'author', 'reviewer', 'last_comment_time', 'comment'])
# self.csv_write.writerow(a)
try:
# next_url = 'https' + el_html.xpath('//a[@class="next pagination-item "]/@href')
next_url = 'https:' + el_html.xpath('//a[contains(text(),"下一页")]/@href')[0]
except:
next_url = None
return data_list, next_url, header
def save_data(self, data_list, header):
for data in data_list:
print(data)
try:
data['last_comment_time'] = data['last_comment_time'].replace('\r', '').replace('\n', '').strip()
data['comment'] = data['comment'].strip().replace('\r', '').replace('\n', '')
except:
print('list')
self.csv_write.writerow(
[data['title'], data['href'], data['author'], data['reviewer'], data['last_comment_time'],
data['comment']])
def run(self):
# url
# headers
next_url = self.url
while True:
# 发送请求，获取响应
data = self.get_data(next_url)
# 从响应中提取数据（数据和翻页用的url）
data_list, next_url, a = self.parse_data(data)
self.save_data(data_list, a)
# 判断是否终结
if next_url == None:
break
if __name__ == '__main__':
tieba = Tieba('上海理工大学')
tieba.run()
运行效果：
