爬虫（四）：案例：爬取猫眼电影网站+爬取股吧网站+爬取药品网站

爬取网站的流程

• 确定网站的哪个url是数据的来源
• 简要分析一下网站结构，查看数据存放在哪里
• 查看是否有分页，并解决分页的问题
• 发送请求，查看response.text是否有我们所需要的数据
• 筛选数据

案例一：爬取猫眼电影

爬取目标：爬取前一百个电影的信息

import re, requests, json


class Maoyan:

    def __init__(self, url):
        self.url = url
        self.movie_list = []
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'
        }
        self.parse()

    def parse(self):
        # 爬去页面的代码
        # 1.发送请求，获取响应
        # 分页
        for i in range(10):
            url = self.url + '?offset={}'.format(i * 10)
            response = requests.get(url, headers=self.headers)
            '''
            1.电影名称
            2、主演
            3、上映时间
            4、评分
            '''

            # 用正则筛选数据，有个原则：不断缩小筛选范围。
            dl_pattern = re.compile(r'(.*?)', re.S)
            dl_content = dl_pattern.search(response.text).group()

            dd_pattern = re.compile(r'(.*?)', re.S)
            dd_list = dd_pattern.findall(dl_content)
            # print(dd_list)
            movie_list = []
            for dd in dd_list:
                print(dd)
                item = {}
                # ------------电影名字
                movie_pattern = re.compile(r'title="(.*?)" class=', re.S)
                movie_name = movie_pattern.search(dd).group(1)
                # print(movie_name)
                actor_pattern = re.compile(r'
(.*?)
', re.S)
                actor = actor_pattern.search(dd).group(1).strip()
                # print(actor)
                play_time_pattern = re.compile(r'
(.*?)：(.*?)
', re.S)
                play_time = play_time_pattern.search(dd).group(2).strip()
                # print(play_time)

                # 评分
                score_pattern_1 = re.compile(r'(.*?)', re.S)
                score_pattern_2 = re.compile(r'(.*?)', re.S)
                score = score_pattern_1.search(dd).group(1).strip() + score_pattern_2.search(dd).group(1).strip()
                # print(score)
                item['电影名字：'] = movie_name
                item['主演：'] = actor
                item['时间：'] = play_time
                item['评分：'] = score
                # print(item)
                self.movie_list.append(item)
                # 将电影信息保存到json文件中
            with open('movie.json', 'w', encoding='utf-8') as fp:
                json.dump(self.movie_list, fp)


if __name__ == '__main__':
    base_url = 'https://maoyan.com/board/4'
    Maoyan(base_url)

    with open('movie.json', 'r') as fp:
        movie_list = json.load(fp)
    print(movie_list)

案例二：爬取股吧

爬取目标： 爬取前十页的阅读数,评论数,标题,作者,更新时间,详情页url

import json
import re

import requests


class GuBa(object):
    def __init__(self):
        self.base_url = 'http://guba.eastmoney.com/default,99_%s.html'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'
        }
        self.infos = []
        self.parse()

    def parse(self):
        for i in range(1, 13):
            response = requests.get(self.base_url % i, headers=self.headers)

            '''阅读数,评论数,标题,作者,更新时间,详情页url'''
            ul_pattern = re.compile(r'
(.*?)
', re.S)
            ul_content = ul_pattern.search(response.text)
            if ul_content:
                ul_content = ul_content.group()

            li_pattern = re.compile(r'
(.*?)
', re.S)
            li_list = li_pattern.findall(ul_content)
            # print(li_list)

            for li in li_list:
                item = {}
                reader_pattern = re.compile(r'(.*?)', re.S)
                info_list = reader_pattern.findall(li)
                # print(info_list)
                reader_num = ''
                comment_num = ''
                if info_list:
                    reader_num = info_list[0].strip()
                    comment_num = info_list[1].strip()
                print(reader_num, comment_num)
                title_pattern = re.compile(r'title="(.*?)" class="note">', re.S)
                title = title_pattern.search(li).group(1)
                # print(title)
                author_pattern = re.compile(r'target="_blank">(.*?)