好久不更爬虫与数据分析系列的文章了
最近学妹拜托我帮她完成了一份爬虫大作业
借着这个机会，更一期的爬虫与数据分析

主题是非常常见的爬取豆瓣读书与豆瓣电影

博主发现一个非常有意思的现象，去年学习爬虫的时候，练手豆瓣读书与豆瓣电影时，爬取还挺容易的，反爬措施也没有那么强~
今天我在爬取的时候，大概只请求了几次，好家伙直接给我IP封了，像这样：

IP代理的话，我还在学习，不太会整，然后用了一个非常巧妙的fake-UserAgent库随机产生请求头，模拟浏览器的请求，欺骗豆瓣的后台，这样果真可以实现数据的爬取

话不多说，直接上代码
豆瓣读书

import re 
import requests
import pandas as pd
from tqdm import tqdm
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

## 定义请求头和url
url = 'https://www.douban.com/doulist/45004834/'
headers = {
    'User-Agent':str(UserAgent().random)
}

def url_get():
    li = []
    url_list = []
    for i in range(0,100,25):
        li.append(i)
    for j in range(len(li)):
        url = 'https://www.douban.com/doulist/45004834/?start={:d}&sort=time&playable=0&sub_type='.format(li[j])
        url_list.append(url)
    return url_list
url_list = url_get()

def write_excel(datalist,path):
    test = pd.DataFrame(data=datalist)
    test.to_excel(path,encoding='gbk')
    return test

info = []
information = []

for i in tqdm(range(0,4)):
    data_list = []
    response = requests.get(url_list[i],headers=headers)
    content = response.content.decode('utf-8')
    
    soup = BeautifulSoup(content,'lxml')
    divs = soup.find_all('div',class_='doulist-item')
    tmp = []
    for div in divs:
        img = div.find('div',class_='post')
        title = div.find('div',class_='title')
        abstract = div.find('div',class_='abstract')
        tmp.append(img)
        tmp.append(title)
        tmp.append(abstract)
        data_list.append(tmp)
        tmp = []


    for j in range(len(data_list)-1):
        img_lianjie = re.findall('src="(.*?)/>',str(data_list[j+1][0]))[0]
        name = re.findall('target="_blank">\n[\s]*(.{2,20})',str(data_list[j+1][1]))[0]
        detail1 = re.findall('abstract">\n[\s]*(.{2,50})',str(data_list[j+1][2]))
        if len(detail1) != 0:
            detail1 = re.findall('abstract">\n[\s]*(.{2,50})',str(data_list[j+1][2]))[0]
        else:
            detail1 = 'None'
        detail2 = re.findall('<br/>\n[\s]*(.{2,50})',str(data_list[j+1][2]))
        info.append(name)
        info.append(detail1)
        info.append(detail2)
        info.append(img_lianjie)
        information.append(info)
        info = []


        
path = 'D:/doubandushu.xlsx'
write_excel(information, path)
      

最后的运行结果如下图所示：

链接是豆瓣读书书的封面图片，可以整一波下载
豆瓣电影

import re 
import requests
import pandas as pd
from tqdm import tqdm
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

## 定义请求头和url
url = 'https://movie.douban.com/top250'
headers = {
    'User-Agent':str(UserAgent().random)
}

def url_get():
    li = []
    url_list = []
    for i in range(0,275,25):
        li.append(i)
    for j in range(len(li)):
        url = 'https://movie.douban.com/top250?start={:d}&filter='.format(li[j])
        url_list.append(url)
    return url_list

url_list = url_get()

def write_excel(datalist,path):
    test = pd.DataFrame(data=datalist)
    test.to_excel(path,encoding='gbk')
    return test
    


info = []
information = []

for i in tqdm(range(0,10)):
    data_list = []
    response = requests.get(url_list[i],headers=headers)
    content = response.content.decode('utf-8')
    
    soup = BeautifulSoup(content,'lxml')
    
    lis = soup.find_all('li')[-26:]
    tmp = []
    for li in lis:
        film_name = li.find('div',class_='hd')
        film_detail = li.find('div',class_='bd')
        tmp.append(film_name)
        tmp.append(film_detail)
        data_list.append(tmp)
        tmp = []

    for j in range(len(data_list)-1):
        name = re.findall('title">(.*?)<',str(data_list[j+1][0]))[0]
        director = re.findall('导演:(.*?)<br/>',str(data_list[j+1][1]))
        director_name = '导演:'+ re.sub('\s','',director[0])
        time = re.findall('\d+\.?\d*',str(data_list[j+1][1]))[0]
        score =re.findall('average">(.*?)<',str(data_list[j+1][1]))[0]
        introduction = re.findall('inq">(.*?)<',str(data_list[j+1][1]))
        if len(introduction) != 0:
            introduction_detail = introduction[0]
        else:
            introduction_detail = 'None'
        info.append(name)
        info.append(director_name)
        info.append(time)
        info.append(score)
        info.append(introduction_detail)
        information.append(info)
        info = []

path = 'D:/doubandianying.xlsx'
write_excel(information, path)

print('爬取完毕')
        

程序结果运行如图：

比较简单的两部分代码，博主写的不好之处还请多多包涵，代码不懂的地方可以私信我或者评论区互相交流~
共同进步！