您现在的位置是：首页 > Python

当前栏目

python3异步爬虫：asyncio + aiohttp + aiofiles（python经典编程案例）

Python Python3 输入下载异步

2023-03-02 11:48:48 时间

爬取网站为：[http://www.tulishe.com/all][http_www.tulishe.com_all]

# -*- encoding: utf-8 -*-
    import os
    import time
    import asyncio
    import aiohttp
    import aiofiles
    
    from lxml import etree
    from functools import wraps
    from asyncio.proactor_events import _ProactorBasePipeTransport
    
    
    def silence_event_loop_closed(func):
        @wraps(func)
        def wrapper(self, *args, **kwargs):
            try:
                return func(self, *args, **kwargs)
            except RuntimeError as e:
                if str(e) != 'Event loop is closed':
                    raise
        return wrapper
    
    
    _ProactorBasePipeTransport.__del__ = silence_event_loop_closed(_ProactorBasePipeTransport.__del__)
    
    headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36', 'Referer': 'http://www.tulishe.com'}
    
    
    class tulishe:
        def __init__(self):
            self.write_num = 0
    
        async def get_url(self, url):
            async with aiohttp.ClientSession() as client:
                async with client.get(url, headers=headers) as resp:
                    if resp.status == 200:
                        return await resp.text()
    
        async def html_parse(self, html):
            semaphore = asyncio.Semaphore(5)
            html_parse = etree.HTML(html)
            url_list = html_parse.xpath('//div[@class="img"]//a[@rel="bookmark"]/@href')
            tasks = [asyncio.create_task(self.img_parse(url, semaphore)) for url in url_list]
            await asyncio.wait(tasks)
    
        async def img_parse(self, h_url, sem):
            async with sem:
                semaphore = asyncio.Semaphore(5)
                h_html = await self.get_url(h_url)
                h_html_parse = etree.HTML(h_html)
                title = h_html_parse.xpath('//h1[@class="article-title"]/text()')[0]
                img_demo_url = h_html_parse.xpath(
                    '//*[@id="gallery-2"]/div[@class="gallery-item gallery-blur-item"]/img/@src')
                img_url_list = []
                for d_url in img_demo_url:
                    img_url = d_url.split('=')[1].split('&')[0]
                    img_url_list.append(img_url)
                i_u_l = h_html_parse.xpath(
                    '//div[@class="gallery-item gallery-fancy-item"]/a/@href')
                full_list = i_u_l + img_url_list
                index_list = list(range(1, len(full_list) + 1))
                index_dict = dict(zip(full_list, index_list))
                tasks = [asyncio.create_task(self.img_con(i_url, i_num, title, semaphore)) for i_url, i_num in
                         index_dict.items()]
                await asyncio.wait(tasks)
    
        async def img_con(self, url, num, title, semaphore):
            async with semaphore:
                async with aiohttp.ClientSession() as client:
                    async with client.get(url, headers=headers) as resp:
                        if resp.status == 200:
                            img_con = await resp.read()
                            await self.write_img(img_con, num, title)
                        else:
                            print('请求出错，请尝试调低并发数重新下载！！')
    
        async def write_img(self, img_con, num, title):
            title = 'tulise_imgs/' + title
            if not os.path.exists(title):
                os.makedirs(title)
                async with aiofiles.open(title + '/' + f'{num}.jpg', 'wb') as f:
                    print(f'正在下载{title}/{num}.jpg')
                    await f.write(img_con)
                    self.write_num += 1
            else:
                async with aiofiles.open(title + '/' + f'{num}.jpg', 'wb') as f:
                    print(f'正在下载{title}/{num}.jpg')
                    await f.write(img_con)
                    self.write_num += 1
    
        async def main(self, ):
            q_start_num = input('输入要从第几页开始下载（按Entry默认为1）:') or '1'
            start_num = int(q_start_num)
            total_num = int(input('请输入要下载的页数：')) + start_num
            print('*' * 74)
            start_time = time.time()
            for num in range(start_num, total_num + 1):
                url = f'http://www.tulishe.com/all/page/{num}'
                html = await self.get_url(url)
                print('开始解析下载>>>')
                await self.html_parse(html)
            end_time = time.time()
            print(f'本次共下载写真图片{self.write_num}张，共耗时{end_time - start_time}秒。')
    
    
    if __name__ == '__main__':
        run_data = tulishe()
        asyncio.run(run_data.main())

[http_www.tulishe.com_all]: http://www.tulishe.com/all

猜你喜欢

最长无重复子串
写技术博客的一些心得分享
Java 多线程（七）：线程池
Java 多线程（五）：锁（三）
Java 多线程（四）：锁（二）
Java 多线程（三）：锁（一）
Java 多线程（二）：并发编程的三大特性
线性时间非比较类排序
Java 多线程（一）：基础
合并k个已排序的链表
HDFS 高可用分布式环境搭建
合并两个有序数组
连续子数组的最大和
HDFS 分布式环境搭建
容器盛水问题
大数加法
HDFS 伪分布式环境搭建
设计LRU缓存结构
两数之和
使用单调栈来解决的一些问题

zl程序教程

当前栏目

python3异步爬虫：asyncio + aiohttp + aiofiles（python经典编程案例）

相关文章