只需要将img的src的属性值进行解析,提交到管道,管道就会对图片的src进行请求发送获取图片的二进制类型的数据,且还会帮我们进行持久化存储
import scrapy from imgPro.items import ImgproItem class ImgSpider(scrapy.Spider): name = 'img' # allowed_domains = ['www.xxx.com'] start_urls = ['http://pic.netbian.com/4kmeinv/'] def parse(self, response): div_list = response.xpath('//*[@id="main"]/div[3]/ul/li') for div in div_list: #注意:使用伪属性 src = 'http://pic.netbian.com' + div.xpath('./a/img/@src').extract_first() item = ImgproItem() item['src'] = src yield item
from scrapy.pipelines.images import ImagesPipeline import scrapy class ImgproPipeline(ImagesPipeline): #就是可以根据图片地址进行图片数据的请求 def get_media_requests(self, item, info): yield scrapy.Request(item['src']) #指定图片存储的路径 def file_path(self, request, response=None, info=None): imgName = request.url.split('/')[-1] return imgName def item_completed(self, results, item, info): return item #返回给下一个即将被执行的管道类