python 多进程存为csv案例，直接在项目中应用
import requests from fake_useragent import UserAgent from tqdm import tqdm from concurrent.futures import ThreadPoolExecutor import time # 爬取一页数据 def download_one_page(data: dict): url = 'http://www.xinfadi.com.cn/getPriceData.html' headers = { 'user-agent': UserAgent().random, 'referer': 'http://www.xinfadi.com.cn/priceDetail.html' } resp = requests.post(url=url, headers=headers, data=data) data_list = resp.json().get('list') # 保存数据 with open('北京新发地.csv', 'a', encoding='utf-8') as fp: for elem in tqdm(data_list, desc=f'下载第 {data["current"]} 页数据 当前状告码:{resp.status_code}', ascii=True): info = (elem['prodCat'], elem['prodPcat'], elem['prodName'], elem['lowPrice'], elem['avgPrice'], elem['highPrice'], elem['specInfo'] , elem['place'], elem['unitInfo'], elem['pubDate']) fp.write(','.join(info) + '\n') def download_pages(page_start: int, page_end: int, page_limit: int = 20): fp = open('北京新发地.csv', 'w', encoding='utf-8') title = ['一级分类', '二级分类', '品名', '最低价', '平均价', '最高价', '规格', '产地', '单位', '发布日期'] fp.write(','.join(title) + '\n') fp.close() with ThreadPoolExecutor(2048) as t: for i in range(page_start, page_end + 1): data = { 'limit': f'{page_limit}', 'current': f'{i}', 'pubDateStartTime': '', 'pubDateEndTime': '', 'prodPcatid': '', 'prodCatid': '', 'prodName': '' } t.submit(download_one_page, data) if __name__ == '__main__': start_time = time.time() download_pages(page_start=1, page_end=100, page_limit=20) end_time = time.time() print(f'总耗时{end_time - start_time}s')
