Python通过解析网页实现看报程序的方法

本文所述实例可以实现基于Python的查看图片报纸《参考消息》并将当天的图片报纸自动下载到本地供查看的功能，具体实现代码如下：

#coding=gbk
importurllib2
importsocket
importre
importtime
importos

#timeoutinseconds
#timeout=10
#socket.setdefaulttimeout(timeout)
timeout=10
urllib2.socket.setdefaulttimeout(timeout)

home_url="http://www.hqck.net"
home_page=""
try:
home_page_context=urllib2.urlopen(home_url)
home_page=home_page_context.read()

print"Readhomepagefinishd."
print"-------------------------------------------------"
excepturllib2.URLError,e:
printe.code
exit()
except:
printe.code
exit()

reg_str=r"<aclass="item-baozhi"href="/arc/jwbt/ckxx/\d{4}/\d{4}/\w+\.html"rel="externalnofollow"><spanclass.+>.+</span></a>"

news_url_reg=re.compile(reg_str)

today_cankao_news=news_url_reg.findall(home_page)

iflen(today_cankao_news)==0:
print"Cannotfindtoday"snews!"
exit()

my_news=today_cankao_news[0]
print"Latestnewslink="+my_news
print

url_s=my_news.find("/arc/")
url_e=my_news.find(".html")
url_e=url_e+5

print"Linkindex=["+str(url_s)+","+str(url_e)+"]"
my_news=my_news[url_s:url_e]
print"parturl="+my_news

full_news_url=home_url+my_news
print"fullurl="+full_news_url
print

image_folder="E:\\new_folder\\"

if(os.path.exists(image_folder)==False):
os.makedirs(image_folder)
today_num=time.strftime("%Y-%m-%d",time.localtime(time.time()))
image_folder=image_folder+today_num+"\\"
if(os.path.exists(image_folder)==False):
os.makedirs(image_folder)
print"Newsimagefolder="+image_folder
print

context_uri=full_news_url[0:-5]

first_page_url=context_uri+".html"
try:
first_page_context=urllib2.urlopen(first_page_url)
first_page=first_page_context.read()
excepturllib2.HTTPError,e:
printe.code
exit()

tot_page_index=first_page.find("共")
tot_page_index=tot_page_index

tmp_str=first_page[tot_page_index:tot_page_index+10]
end_s=tmp_str.find("页")

page_num=tmp_str[2:end_s]
printpage_num

page_count=int(page_num)
print"Total"+page_num+"pages:"
print

page_index=1
download_suc=True
whilepage_index<=page_count:
page_url=context_uri
ifpage_index>1:
page_url=page_url+"_"+str(page_index)
page_url=page_url+".html"
print"Newspagelink="+page_url

try:
news_img_page_context=urllib2.urlopen(page_url)
excepturllib2.URLError,e:
printe.reason
download_suc=False
break

news_img_page=news_img_page_context.read()

#f=open("e:\\page.html","w")
#f.write(news_img_page)
#f.close()

reg_str=r"http://image\S+jpg"
image_reg=re.compile(reg_str)
image_results=image_reg.findall(news_img_page)
iflen(image_results)==0:
print"Cannotfindnewspage"+str(page_index)+"!"
download_suc=False
break

image_url=image_results[0]

print"Newsimageurl="+image_url
news_image_context=urllib2.urlopen(image_url)

image_name=image_folder+"page_"+str(page_index)+".jpg"
imgf=open(image_name,"wb")
print"Gettingimage..."
try:
whileTrue:
date=news_image_context.read(1024*10)
ifnotdate:
break
imgf.write(date)
imgf.close()
except:
download_suc=False
print"Saveimage"+str(page_index)+"failed!"
print"Unexpectederror:"+sys.exc_info()[0]+sys.exc_info()[1]
else:
print"Saveimage"+str(page_index)+"succeed!"
print
page_index=page_index+1

ifdownload_suc==True:
print"Newsdownloadsucceed!Path=\""+str(image_folder)+"\""
print"Enjoyit!^^"
else:
print"newsdownloadfailed!"