功能概述

通过Python网络爬虫，爬取相关专栏文章，保存内容至本地文件夹，格式为html，PDF，makedown。

效果展示

环境准备：

• python基础环境

• 安装相关库：

  • pip install requests
  • pip install parsel
  • pip install pdfkit
  • pip install html2text

• 安装wkhtmltopdf（下载方式见百度）

运行步骤

clone到本地
导入vscode
按照注释修改代码中指定位置
右键run code运行代码

项目源码

"""
Author:Lorren
Time  : 2022年8月28日10:41:49
Function:Crawl the blog post of blogger csdn and save it in html/pdf/markdown formats
"""
import requests
import parsel
import os
import re
import pdfkit
import html2text


wkhtmltopdf_options = {
    'enable-local-file-access': None
}


def change_title(name):
    mode = re.compile(r'[\\\/\:\*\?\"\<\>\|]')
    newname = re.sub(mode, '_', name)
    return newname


filename = 'html\\'
filename1 = 'pdf\\'
filename2 = 'md\\'
if not os.path.exists(filename):
    os.mkdir(filename)
if not os.path.exists(filename1):
    os.mkdir(filename1)
if not os.path.exists(filename2):
    os.mkdir(filename2)

html_str = """
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Document</title>
</head>
<body>
{article}
</body>
</html>
"""

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36 Edg/104.0.1293.70'
}


url = f'https://blog.csdn.net/qq_41684621/category_9440849.html'
# 此处替换相应想爬取的网址
response = requests.get(url=url, headers=headers)
selector = parsel.Selector(response.text)
href = selector.css(
    '.column_article_list li a::attr(href)').getall()
# 此处换成对应网页的对应属性即可
for i in href:
    response_1 = requests.get(url=i, headers=headers)
    selector_1 = parsel.Selector(response_1.text)
    title = selector_1.css('#articleContentId::text').get()  # 文章标题的id
    newtitle = change_title(title)
    content_views = selector_1.css(
        '#content_views').get()  # 文章内容的id
    html_content = html_str.format(article=content_views)
    html_name = filename+newtitle+'.html'
    pdf_name = filename1+newtitle+'.pdf'
    md_name = filename2 + newtitle+'.md'
    with open(html_name, mode='w', encoding='utf-8') as f:
        f.write(html_content)
    config = pdfkit.configuration(
        wkhtmltopdf=r'D:\HTMLtoPDF\wkhtmltopdf\bin\wkhtmltopdf.exe')
    # 此处安装路径需要更改为你本地的安装路径
    pdfkit.from_file(html_name, pdf_name, configuration=config,
                     options=wkhtmltopdf_options)
    # os.remove(html_name)
    # 将html文件转换成md格式
    html_text = open(html_name, 'r', encoding='utf-8').read()
    markdown = html2text.html2text(html_text)
    with open(md_name, 'w', encoding='utf-8') as file:
        file.write(markdown)
        print("正在保存", newtitle, ".md")