爬虫伪装

以豆瓣为例，代码如下

import urllib.request
import urllib.parse

url = 'https://www.douban.com'
response = urllib.request.urlopen(url)

print(response.read().decode('utf-8'))

会返回418错误

---

封装一个请求头，设置user-agent字段就可以模拟成浏览器

import urllib.request
import urllib.parse
import urllib.error


def main():
  url = 'https://movie.douban.com/top250?start='
  askURL(url)


def askURL(url):
# 做测试只修改user agent字段
  head = {
    "User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Mobile Safari/537.36"}
  res = urllib.request.Request(url=url, headers=head)
  html = ""
  try:
    response = urllib.request.urlopen(res)
    html = response.read().decode('utf-8')
    print(html)
  except urllib.error.URLError as e:
    if hasattr(e, "code"):
      print(e.code)
    if hasattr(e, 'reason'):
      print(e.reason)


if __name__ == "__main__":
  main()

结果如下：