【Python爬虫案例学习19】多进程爬取某图片网站

网站:http://www.win4000.com（图片的质量还不错）

基本环境配置：

• Python版本：3.6

相关模块：

import re
import requests
import time
from multiprocessing import Pool
from lxml import etree
import os
import uuid

####完整代码：

import re
import requests
import time
from multiprocessing import Pool
from lxml import etree
import os
import uuid
'''
遇到不懂的问题？Python学习交流群：821460695满足你的需求，资料都已经上传群文件，可以自行下载！
'''
# 第一个主页面地址
rooturl = 'http://www.win4000.com/zt/huyan_'

# http://www.win4000.com/zt/fengjing.html

# 模拟浏览器请求头
header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
                  " AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/70.0.3538.110 Safari/537.36"
}

count = 0
# 图片集url
def graph_set(rooturl):
    set = []
    title = []
    results = requests.get(rooturl, headers=header)
    text = results.text
    res = re.findall('.*href="(.*)" alt="',text)
    selector = etree.HTML(text)
    tt = selector.xpath('//div[contains(@class,"tab_tj")]//li//p')
    for url in res:
        set.append(url)
    for tit in tt[:24]:
        title.append(tit.text)
    return title,set


# 图片页面解析原图集合
def parser(tup):
    response = requests.get(tup[0],headers=header)
    text = response.text
    originset = re.findall('href="(.*)" class=.*查看原图',text)
    time.sleep(1)
    oringin(originset.pop(),tup[1])

# 图集原图集合
def oringin(page,name):
    print(name+'正在爬取')
    dir = 'G:\python 资源\python project\美桌网壁纸爬取\护眼图片\\'
    oringin = []
    response = requests.get(page,headers=header)
    res = re.findall('li.*href="(.*)".*>