Python爬虫之json动态数据抓取

本文主要是介绍Python爬虫之json动态数据抓取，对大家解决编程问题具有一定的参考价值，需要的程序猿们随着小编来一起学习吧！

python爬虫之get请求

# 安装requests包：pip install requests
import requests

response = requests.get('https://www.baidu.com/more/')
print(response)  # <Response [200]>

headers = {
    # 浏览器类型
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36'
}
print(response.content.decode('utf-8'))

python爬虫之post请求

# post 请求
import requests

url = 'https://fanyi.baidu.com/langdetect'

form_data = {
    'query':'爬虫'
}

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36'
}

response = requests.post(url,data=form_data,headers = headers)
print(response.json())

python爬虫之xpath数据提取

'''
xpath语法
//   -> 跟节点
/    -> 节点
@    -> 属性
'''

import requests
from lxml import etree

url = 'https://www.cnblogs.com/xyxuan/p/14336276.html'
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36'
}
# 获取网页数据
response = requests.get(url,headers = headers).content.decode('utf-8')
# 提取网页图片地址
html = etree.HTML(response)
img_url = html.xpath('//p/img/@src')
print(img_url)

json动态数据抓取

好啦，实战开始！！！

直接上源码，以爬取51Job的职位信息为例，可以根据自己需要抓取的网站替换 URL & headers

"""
@ Time:2021/06/15
利用python爬取职位信息，并写入CSV文件中
* 数据来源: 51job
* 开发环境：win10、python3.9
* 开发工具：pycharm、Chrome

项目思路：
1. 首先列出需要获取的数据信息：['职位名称', '公司名称', '薪资','工作地点','公司类型','其他']
2. 对页面数据进行解析，对页面数据进行JSON格式化，找出所需要的字段，并根据需要进行分类
3. 将获取的数据写入到CSV文件中【注：运行代码时，CSV文件不能是打开的状态，不然会报文件找不到】
"""

import requests
import time
import csv


def inner():
    """
    简单的装饰器，主要用于console窗口打印或者写入到TXT文件时，看起来一目了然
    :return:
    """
    print('******************************************\n')


def RequestsTools(url, headers):
    """
    爬虫请求工具参数
    :param headers: 请求头
    :param url: 请求地址
    :return: json格式数据
    """
    response = requests.get(url=url, headers=headers).json()
    return response


def Index(page):

    """
    根据页数查询数据函数
    :param page:页数

    """
    print('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@  第' + str(page) + '页  @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n')

    # 请求头
    headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Cookie': 'adv=adsnew%3D1%26%7C%26adsnum%3D7093970%26%7C%26adsresume%3D1%26%7C%26adsfrom%3Dhttps%253A%252F%252Fwww.baidu.com%252Fother.php%253Fsc.Kf00000T8huPAl74RAWWief16RqsEaFV3GEpSPlG_RH0OAZZM913BX4XW42Rq9wSaxNCEdrFH59E8nm_o2Blo0QXlwIwEzKH3XKfx9bWUnDd6UIPrw7IjCBTkDu4QJYRQsPbuYCnHiw1COgZeWIT1Q5-pquOVZXVfbnqRTyPXgbOsPGr-2-rTiM16KOck4fyM8ye8sI3nKQch1AbJ0l52PBRzExL.DR_NR2Ar5Od66CHnsGtVdXNdlc2D1n2xx81IZ76Y_XPhOWEtUrorgAs1SOOo_9OxOBI5lqAS61kO56OQS9qxuxbSSjO_uPqjqxZOg7SEWSyWxSrOSFO_OguCOBxQetZO03x501SOOoCgOQxG9YelZ4EvOqJGMqEOggjSS4Wov_f_lOA7MuvyNdleQeAI1PMAeB-5Wo9Eu88lN2s1f_TTMHYv00.TLFWgv-b5HDkrfK1ThPGujYknHb0THY0IAYqPH7JUvc0IgP-T-qYXgK-5H00mywxIZ-suHY10ZIEThfqPH7JUvc0ThPv5HD0IgF_gv-b5HDdnWT1rHcLrj60UgNxpyfqnHfzP1f3rHD0UNqGujYknjb1rj0LP6KVIZK_gv-b5HDznWT10ZKvgv-b5H00pywW5R420ZKGujYz0APGujY1rHm0mLFW5HcvPH6s%2526ck%253D2896.2.223.275.159.314.188.588%2526dt%253D1623736783%2526wd%253D51job%2526tpl%253Dtpl_12273_25609_20875%2526l%253D1527392788%2526us%253DlinkName%25253D%252525E6%252525A0%25252587%252525E9%252525A2%25252598-%252525E4%252525B8%252525BB%252525E6%252525A0%25252587%252525E9%252525A2%25252598%252526linkText%25253D%252525E3%25252580%25252590%252525E5%25252589%2525258D%252525E7%252525A8%2525258B%252525E6%25252597%252525A0%252525E5%252525BF%252525A751Job%252525E3%25252580%25252591-%25252520%252525E5%252525A5%252525BD%252525E5%252525B7%252525A5%252525E4%252525BD%2525259C%252525E5%252525B0%252525BD%252525E5%2525259C%252525A8%252525E5%25252589%2525258D%252525E7%252525A8%2525258B%252525E6%25252597%252525A0%252525E5%252525BF%252525A7%2521%252526linkType%25253D%26%7C%26; guid=fa4573e0cb7f90a6371f267812a399a8; slife=lowbrowser%3Dnot%26%7C%26; nsearch=jobarea%3D%26%7C%26ord_field%3D%26%7C%26recentSearch0%3D%26%7C%26recentSearch1%3D%26%7C%26recentSearch2%3D%26%7C%26recentSearch3%3D%26%7C%26recentSearch4%3D%26%7C%26collapse_expansion%3D; search=jobarea%7E%60020000%7C%21ord_field%7E%600%7C%21recentSearch0%7E%60020000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%C8%ED%BC%FE%B2%E2%CA%D4%B9%A4%B3%CC%CA%A6%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36'
    }

    url = 'https://search.51job.com/list/020000,000000,0000,00,9,99,%25E8%25BD%25AF%25E4%25BB%25B6%25E6%25B5%258B%25E8%25AF%2595%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,' \
          '2,{}.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='.format(page)

    response = RequestsTools(url,headers)

    # 需要获取的信息
    information = ['job_name', 'company_name', 'providesalary_text', 'workarea_text', 'companytype_text','attribute_text']

    # top 广告位
    def top():
        list = []
        data = ['top_ads', 'auction_ads', 'market_ads']
        for top in data:
            if len(response[top]):
                for i in information:
                    content = response[top][0][i]
                    list.append(content)
                print(list)
                write(list)
                list = []
                inner()
                time.sleep(6) # 让他睡一会吧，不然查询太过频繁IP被封了

            else:
                print(top + '广告位为空！！！\n')

    # 正常广告位
    def ordinarys():
        list = []
        ordinarys = response['engine_search_result']
        for ordinary in ordinarys:
            for i in information:
                content = ordinary[i]
                list.append(content)
            print(list)
            write(list)
            list = []
            inner()
            time.sleep(6)

    if page == 1:
        top()
        ordinarys()
    else:
        ordinarys()


# 写入数据到CSV文件中
def write(list):
    path = r'D:\learn\project\reptile\51job.csv'
    with open(path, mode="a", encoding="gbk", newline='') as f:

        # 基于文件对象构建CSV写入对象
        csv_writer = csv.writer(f)
        # 将构建好的数组依次写入表格，每次调用会自动换行
        csv_writer.writerow(list)

        # 关闭文件
        f.close()

if __name__ == '__main__':
    for page in range(0,11):
        page += 1
        Index(page)

console窗口显示：

写入到CSV文件显示：

同样的，也可以写入的txt文件中，且不需要转换为List 格式，直接str格式即可写入，样式自己调整~

import requests

"""
需要获取的信息
"""
information = ['job_name', 'company_name', 'providesalary_text', 'workarea_text', 'companytype_text', 'attribute_text']

"""
装饰器
"""
def inner():
    print('******************************************\n')

def url(num):
    print('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@  第'+str(num)+'页  @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n')
    wfile('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@  第'+str(num)+'页  @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n')
    # 动态url
    url = 'https://search.51job.com/list/020000,000000,0000,32,9,99,%25E8%25BD%25AF%25E4%25BB%25B6%25E6%25B5%258B%25E8%25AF%2595%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,2,'+ str(num) +'.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='
    headers = {
        # 请求类型
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Referer': 'https://search.51job.com/list/020000,000000,0000,32,9,99,%25E8%25BD%25AF%25E4%25BB%25B6%25E6%25B5%258B%25E8%25AF%2595%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,2,'+ str(num) +'.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36'
    }

    # 建立一个session，用来存储cookies
    # session = requests.session().get(url,headers = headers)
    # cokie = session.cookies
    response = requests.get(url=url,headers=headers).json()

    # top 广告位
    def top():
        data = ['top_ads','auction_ads','market_ads']
        for top in data:
            if len(response[top]):
                for i in information:
                    content = response[top][0][i]
                    print(str(content))
                    wfile(str(content))
                    wfile('\n')
                inner()
                wfile('******************************************\n')
            else:
                print(top + '广告位为空！！！\n')
                wfile(top + '广告位为空！！！\n')
                inner()
                wfile('******************************************\n')

    # 正常广告位
    def ordinarys():
        ordinarys = response['engine_search_result']
        for ordinary in ordinarys:

            for i in information:
                content = ordinary[i]
                print(content)
                wfile(str(content))
                wfile('\n')
            inner()
            wfile('******************************************\n')

    if num == 1:
        top()
        ordinarys()
    else:
       ordinarys()


# 将文件写入到excel中  ---['w':覆盖写模式，‘a’：追加写模式]
def wfile(reptile):
    with open("/Users/yexuan/python/python_reptile/information.txt", mode="a", encoding="utf-8") as f:
        for line in reptile:
            f.write(line)


if __name__ == '__main__':
    number = 1
    while number <= 10:
        url(number)
        number += 1

bingo~

下一步计划，将爬出的数据通过数据可视化图形显示@.@

这篇关于Python爬虫之json动态数据抓取的文章就介绍到这儿，希望我们推荐的文章对大家有所帮助，也希望大家多多支持为之网！

Javascript

Python爬虫之json动态数据抓取

python爬虫之get请求

python爬虫之post请求

python爬虫之xpath数据提取

json动态数据抓取

console窗口显示：

写入到CSV文件显示：

前端开发

后端开发

移动端开发

数据库

服务器运维

人工智能

区块链

游戏开发

网站运营

大数据/云计算

软件工程

软件/开发工具使用

资讯