pip install selenium
如果你使用的是 anaconda , 使用这个打开命令行,并且执行上面语句
chromedriver
https://chromedriver.chromium.org/downloads
下载的文件保存到 c:/chromedriver/chromedriver.exe
下面为爬取天眼查的公司例子
打开 py 编辑器
from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By import pandas as pd import time # chrome driver 驱动 path = "C:/chromedriver/chromedriver.exe" # 创建 chrome driver 实例 driver = webdriver.Chrome(service=Service(path)) # 进入主页 driver.get("https://www.tianyancha.com/login") time.sleep(1) # 跳转到账号密码登录 driver.execute_script("loginObj.changeCurrent(1);") time.sleep(1) # 找到 #mobile 元素, 并输入账号 driver.find_element(By.ID,"mobile").send_keys("xxxxxxxx") # 找到 #password 元素, 输入密码 driver.find_element(By.ID,"password").send_keys("xxxxxxxx") # 使用 By.CSS_SELECTOR , 进行css选择器搜索,并点击登录框 driver.find_element(By.CSS_SELECTOR,'[οnclick="loginObj.loginByPhone(event);"]').click() # 等待 10 秒钟,在这期间手动进行滑动块的验证 time.sleep(10) # 爬取的页数 count = 5 # 爬取结果 companyList = [] # 爬取前 5 页数据 for i in range(1,5): # 遍历页数 driver.get("https://www.tianyancha.com/search/p"+str(i)+"?key=北京企业") results = driver.find_elements(By.CLASS_NAME,"search-result-single") for res in results: # 创建一个字典 obj = {} obj['公司名字'] = res.find_element(By.CSS_SELECTOR,".name").text obj['注册资本'] = res.find_element(By.CSS_SELECTOR,".title.-narrow.text-ellipsis").text.replace('注册资本:', '') obj['成立日期'] = res.find_elements(By.CSS_SELECTOR,".title.text-ellipsis")[2].text.replace('成立日期:', '') obj['公司评分'] = res.find_element(By.CSS_SELECTOR,".score").text .replace("分","") # 将字典添加到列表 companyList.append(obj) # 保存数据 df = pd.DataFrame(companyList) df.to_csv("C:/result.csv",index=0,encoding="gbk")