爬点图片做文章封面，只要封面就行了，还免去了压缩的烦恼，

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
# -*- encoding: utf-8 -*-
'''
@file      :spider.py
@Time      :2021-02-05 01:02:34
@Author    :Karcyril
@Software  :Visual Studio Code
'''

from btkrc import myModule as h
import requests
from bs4 import BeautifulSoup
import re
import os

basePath = 'yourPath'
baseUrl = 'http://pic.netbian.com'


def main():
    url = baseUrl+'/4kdongman/index.html'

    for pageIndex in range(1, 100):
        soup = BeautifulSoup(askUrl(url).text, 'html.parser')

        getImgs(soup)

        url = getNextUrl(soup)


def checkDir():
    if not os.path.exists(basePath):
        os.makedirs(basePath)


def getImgs(soup):
    lis = soup.select('#main > .slist > ul > li')
    checkDir()
    for li in lis:
        if(re.match(r'<li>.*?</li>', str(li))):
            imgUrl = baseUrl+li.a.img['src']
            fileName = imgUrl.split('/')[-1]
            with open(basePath+'/'+fileName, 'wb+') as fp:
                fp.write(askUrl(imgUrl).content)


def getNextUrl(soup):
    navs = soup.select('#main > div.page')[0]
    pNextPage = re.compile(r'<a href="([^<>]*?)">下一页</a>')
    url = baseUrl+pNextPage.search(str(soup)).group(1)
    return url


def askUrl(url):
    i = 0
    while(i < 5):
        try:
            resp = requests.get(url=url, headers=h.headers, timeout=15)
            resp.raise_for_status()
            # 按GBK编码
            resp.encoding = 'gbk'
            return resp
        except requests.exceptions.HTTPError as e:
            print('%s\n重试...%d' % (e, i+1))
        except requests.exceptions.RequestException as e:
            print('%s\n重试...%d' % (e, i+1))

        i += 1


if __name__ == "__main__":
    main()