Python爬虫实战(一):爬糗事百科段子
2023-09-14 09:00:35 时间
代码:
# _*_ coding:utf-8 _*_ import urllib2 import re from datetime import datetime class QSBK: def __init__(self): self.pageIndex = 1 self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)' self.headers = {'User-Agent':self.user_agent} self.stories = [] self.enable = False def getPage(self,pageIndex): try: url = 'http://www.qiushibaike.com/hot/page'+str(pageIndex) request = urllib2.Request(url,headers = self.headers) response = urllib2.urlopen(request) pageCode = response.read().decode('utf-8') return pageCode except urllib2.URLError,e: if hasattr(e,'reason'): print u"QSBK connect Error,reason: ",e.reason return None def getPageItems(self,pageIndex): pageCode = self.getPage(pageIndex) if not pageCode: print "Page Loading Error..." return None pattern = re.compile('<div.*?author clearfix">.*?<a.*?<img.*?>(.*?)</a>.*?<a.*?<h2>(.*?)</h2>.*?</a>.*?<div.*?'+'content">(.*?)<!--(.*?)-->.*?</div>.*?<div class="stats.*?class="number">(.*?)</i>',re.S) items = re.findall(pattern,pageCode) pageStories = [] for item in items: haveImg = re.search("img",item[0]) if not haveImg: replaceBR = re.compile('<br/>') text = re.sub(replaceBR,"\n",item[2]) pageStories.append([item[1].strip(),text.strip(),item[3].strip(),item[4].strip()]) return pageStories def loadPage(self): if self.enable == True: if len(self.stories) < 2: pageStories = self.getPageItems(self.pageIndex) if pageStories: self.stories.append(pageStories) self.pageIndex += 1 def getOneStory(self,pageStories,page): for story in pageStories: input = raw_input() self.loadPage() if input == 'Q': self.enable = False return print u"第%d页\t发布人:%s\t发布时间:%s\t赞:%s\n%s" %(page,story[0],datetime.fromtimestamp(int(story[2])),story[3],story[1]) def start(self): print u"正在读取糗事百科,按回车查看新段子,Q退出" self.enable = True self.loadPage() nowPage = 0 while self.enable: if len(self.stories)>0: pageStories = self.stories[0] nowPage += 1 del self.stories[0] self.getOneStory(pageStories,nowPage) spider = QSBK() spider.start()
相关文章
- 遗传算法做多目标优化_python 遗传算法
- Python怎么输入小数和整数_python输入非负整数
- python中pygame怎么安_Python中pygame安装方法图文详解
- Python - 手把手教你用Scrapy编写一个爬虫
- python进制转换函数-Python中进制转换函数的使用
- python chr()和ord()_Python函数ord
- Python 协程 asyncio 极简入门与爬虫实战
- 【说站】python系统内置方法如何获取
- Python GUI神器Gooey
- Python招聘岗位信息聚合系统(拥有爬虫爬取、数据分析、可视化、互动等功能)
- python中if判断语句的用法_Python if判断语句的用法详细介绍[通俗易懂]
- 【Python】字符串 ⑤ ( Python 字符串快速格式化 | 不考虑变量类型 | 不考虑精度控制 )
- Python实现文件的压缩及解压详解编程语言
- 怎么避免Python程序出现死循环(无限循环)?
- python使用os模块的os.walk遍历文件夹示例
- python实现倒计时的示例
- python编写网页爬虫脚本并实现APScheduler调度
- Python实现二分法算法实例
- python爬虫入门教程之糗百图片爬虫代码分享