python爬虫控制aiohttp并发数量方式

  
  
  
  
  
  

   
   
   
   
   
   

    from asyncio import tasks
   
   
   
   
   
   
   
   
   
   
   
   

    from aiohttp.client import ClientSession
   
   
   
   
   
   
   
   
   
   
   
   

    from lxml import etree
   
   
   
   
   
   
   
   
   
   
   
   

    from time import sleep
   
   
   
   
   
   
   
   
   
   
   
   

    import time
   
   
   
   
   
   
   
   
   
   
   
   

    import asyncio
   
   
   
   
   
   
   
   
   
   
   
   

    import aiohttp
   
   
   
   
   
   
   
   
   
   
   
   

     
   
   
   
   
   
   
   
   
   
   
   
   

    async def read_page_list(page_num,sem):
   
   
   
   
   
   
   
   
   
   
   
   

        params = {
   
   
   
   
   
   
   
   
   
   
   
   

            'page':page_num,
   
   
   
   
   
   
   
   
   
   
   
   

        }
   
   
   
   
   
   
   
   
   
   
   
   

        #通过连接池控制并发数量 limit 默认为100  0 为无限制
   
   
   
   
   
   
   
   
   
   
   
   

        async with sem:
   
   
   
   
   
   
   
   
   
   
   
   

            try:
   
   
   
   
   
   
   
   
   
   
   
   

                async with aiohttp.ClientSession() as session:
   
   
   
   
   
   
   
   
   
   
   
   

                    async with session.get(url=url,params=params,headers=headers) as response:
   
   
   
   
   
   
   
   
   
   
   
   

                        text = await response.text()
   
   
   
   
   
   
   
   
   
   
   
   

            except Exception as e:
   
   
   
   
   
   
   
   
   
   
   
   

                print('exception:',e)
   
   
   
   
   
   
   
   
   
   
   
   

             
   
   
   
   
   
   
   
   
   
   
   
   

            tree = etree.HTML(text)
   
   
   
   
   
   
   
   
   
   
   
   

            page_list = tree.xpath('//*[@id="thumbs"]/section[1]/ul/li')
   
   
   
   
   
   
   
   
   
   
   
   

            # break
   
   
   
   
   
   
   
   
   
   
   
   

            for li in page_list:
   
   
   
   
   
   
   
   
   
   
   
   

                pic_small_url = li.xpath('.//img/@data-src')[0]
   
   
   
   
   
   
   
   
   
   
   
   

                # print(pic_small_url,type(pic_small_url))
   
   
   
   
   
   
   
   
   
   
   
   

                # pic_small_url = str(pic_small_url)
   
   
   
   
   
   
   
   
   
   
   
   

                if 'small' in pic_small_url:
   
   
   
   
   
   
   
   
   
   
   
   

                    temp_url = pic_small_url.replace('small','full')
   
   
   
   
   
   
   
   
   
   
   
   

                    a = temp_url.rfind('/')
   
   
   
   
   
   
   
   
   
   
   
   

                    temp_url1= temp_url[:a]
   
   
   
   
   
   
   
   
   
   
   
   

                    pic_full_url = temp_url1+'/wallhaven-'+temp_url.split('/')[-1]
   
   
   
   
   
   
   
   
   
   
   
   

                    pic_full_url = pic_full_url.replace('th','w')
   
   
   
   
   
   
   
   
   
   
   
   

                    # print(page_num,pic_full_url)
   
   
   
   
   
   
   
   
   
   
   
   

                    pic_list.append(pic_full_url)
   
   
   
   
   
   
   
   
   
   
   
   

                else:
   
   
   
   
   
   
   
   
   
   
   
   

                    print(page_num,'find small error',pic_small_url)
   
   
   
   
   
   
   
   
   
   
   
   

                 
   
   
   
   
   
   
   
   
   
   
   
   

            print(page_num,len(page_list),response.status)
   
   
   
   
   
   
   
   
   
   
   
   

            # await asyncio.sleep(1)
   
   
   
   
   
   
   
   
   
   
   
   

            #这里可以用硬延时来控制程序的访问速度，进而控制单位时间内并发的数量
   
   
   
   
   
   
   
   
   
   
   
   

            # sleep(0.5)
   
   
   
   
   
   
   
   
   
   
   
   

     
   
   
   
   
   
   
   
   
   
   
   
   

    #定义信号量
   
   
   
   
   
   
   
   
   
   
   
   

    sem = asyncio.Semaphore(2)
   
   
   
   
   
   
   
   
   
   
   
   

     
   
   
   
   
   
   
   
   
   
   
   
   

    start = time.time()
   
   
   
   
   
   
   
   
   
   
   
   

    #建立任务列表
   
   
   
   
   
   
   
   
   
   
   
   

    tasks = [loop.create_task(read_page_list(i,sem)) for i in range(1,20)]
   
   
   
   
   
   
   
   
   
   
   
   

    loop.run_until_complete(asyncio.wait(tasks))
   
   
   
   
   
   
   
   
   
   
   
   

    print('get page list use time:',time.time()-start)