asyncio官网
https://docs.python.org/zh-cn/3/library/asyncio-task.html
下面为伪代码:
import aiohttpimport asyncio
from bs4 import BeautifulSoupimport pandas as pd# 将数据存入 li=[]或数据库#获取页面li=[]
async def fetch(url,session): async with session.get(url) as response: return await response.text# 解析网页async def parse(html): soup = BeautifulSoup(html,'lxml') # 获取网页中的畅销书
book_list=soup.find('ul',class_='book_list')('li') for book in book_list:
info =book.find_all('div')
# 获取每本畅销书的排名,名称,评论数,作者,出版社
rank = info[0].text[0:-1]
name = info[2].text
comments = info[3].text.split('条')[0]
author = info[4].text
date_and_publisher = info[5].text.split()
publisher = date_and_publisher[1] if len(date_and_publisher) >= 2 else '' # 将每本畅销书的上述信息加入到table中
li.append([rank, name, comments, author, publisher])# 处理页面async def download(url): async with aiohttp.ClientSession as session: # 获取页面
html = await fetch(session, url) # 解析页面
await parse(html)# 全部网页urlsurls=[url1,url2,url3]# 利用asycio 模块进行一步IO处理
loop = asyncio.get_event_loop()# 异步获取任务
tasks= [asyncio.ensure_future(download(url)) for url in urls]tasks = asyncio.gather(*tasks)loop.run_until_complete(tasks)# 将table转化为pandas中的DataFrame并保存为CSV格式的文件
df = pd.DataFrame(li, columns=['rank','name','comments','author','publisher'])df.to_csv('E://douban/dangdang.csv',index=False)