Python: “股票数据定向爬虫”实例

import requests, re, traceback
from bs4 import BeautifulSoup

def getHTMLText(url, code="utf-8"):
    try:
        r = requests.get(url)
        r.raise_for_status()
        r.encoding = code
        return r.text
    except Exception as exc:
        print('\nThere was a problem: %s' % (exc))
        return ""
    
def getStockList(lst, stockURL):
    html = getHTMLText(stockURL, "GB2312")
    soup = BeautifulSoup(html, 'html.parser') 
    a = soup.find_all('a')
    for i in a:
        try:
            href = i.attrs['href']
            temp = re.findall(r"/stock/\d{6}/", href)[0]
            if temp[7] == "6":
                lst.append("SH" + temp[7:13]) 
            else:
                lst.append("SZ" + temp[7:13])
        except:
            continue

def getStockInfo(lst, stockURL, fpath):
    count = 0
    for stock in lst:
        url = stockURL + stock
        html = getHTMLText(url)
        try:
            if html=="":
                count = count + 1
                print("\r当前进度: {:.2f}%".format(count*100/len(lst)),end="")
                continue
            infoDict = {}
            soup = BeautifulSoup(html, 'html.parser')
            stockInfo1 = soup.find('header',attrs={'class':'stock_title'})
            infoDict.update({'股票名称': stockInfo1.h1.string})
            
            stockInfo2 = soup.find('section',attrs={'class':'stock_price clearfix'})
            
            keyList = stockInfo2.find_all('dt')
            valueList = stockInfo2.find_all('dd')
            for i in range(len(keyList)):
                key = keyList[i].text
                val = valueList[i].text
                infoDict[key] = val
            
            with open(fpath, 'a', encoding='utf-8') as f:
                f.write( str(infoDict) + '\n' )
                count = count + 1
                print("\r当前进度: {:.2f}%".format(count*100/len(lst)),end="")
        except:
            count = count + 1
            print("\r当前进度: {:.2f}%".format(count*100/len(lst)),end="")
            traceback.print_exc()
            continue

def main():
    stock_list_url = 'http://www.cgedt.com/stockcode/yilanbiao.asp'
    stock_info_url = 'https://hq.gucheng.com/'
    output_file = 'E://python123//GuchengStockInfo.txt'
    
    slist=[]
    getStockList(slist, stock_list_url)
    getStockInfo(slist, stock_info_url, output_file)

main()