Python爬虫 —— 抓取美女图片（Scrapy篇）

 

 

 

杂谈：

之前用requests模块爬取了美女图片，今天用scrapy框架实现了一遍。

（图片尺度确实大了点，但老衲早已无恋红尘，权当观赏哈哈哈）

 

Item:

# -*- coding: utf-8 -*-# Define here the models for your scraped items
#
# See documentation in:
# .html
import scrapyclass GirlpicItem(scrapy.Item):title = scrapy.Field()image = scrapy.Field()index = scrapy.Field()

 

Spider:

#coding:utf-8
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.selector import Selector
from girlpic.items import GirlpicItem
import scrapy
import sys
reload(sys)
sys.setdefaultencoding('utf-8')class GirlpicSipder(Spider):name = 'girlpic'allowed_domains = []  # 允许的域名start_urls = ["/"]def parse(self, response):groups = response.xpath("//div[@class='main-content']//ul[@class='archives']//a")count = 0for group in groups:count = count + 1if count > 5:return   #此处小心，不要用os.exit(0)groupUrl = group.xpath('@href').extract()[0]title = group.xpath("text()").extract()[0]request = scrapy.Request(url=groupUrl, callback=self.getGroup, meta={'title': title,'groupUrl':groupUrl}, dont_filter=True)yield requestdef getGroup(self, response):maxIndex = response.xpath("//div[@class='pagenavi']//span/text()").extract()[-2]for index in range(1, int(maxIndex) + 1):pageUrl = response.meta['groupUrl']+'/'+str(index)meta = response.metameta['index'] = indexrequest = scrapy.Request(url=pageUrl, callback=self.getPage, meta=meta, dont_filter=True)yield requestdef getPage(self, response):imageurl = response.xpath("//div[@class='main-image']//img/@src").extract()[0]  # 获取图片urlrequest = scrapy.Request(url=imageurl, callback=self.FormItem, meta=response.meta,dont_filter=True)yield requestdef FormItem(self, response):title = response.meta['title']index = response.meta['index']image = response.bodyitem = GirlpicItem(title=title,index=index,image=image)yield item

 

 

PipeLine:

# -*- coding: utf-8 -*-# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: .htmlimport os
import codecs
import sys
reload(sys)
sys.setdefaultencoding('utf-8')class GirlpicPipeline(object):def __init__(self):self.dirpath = u'D:\学习资料'if not os.path.exists(self.dirpath):os.makedirs(self.dirpath)def process_item(self, item, spider):title = item['title']index = item['index']image = item['image']groupdir = os.path.join(self.dirpath, title)if not os.path.exists(groupdir):os.makedirs(groupdir)imagepath = os.path.join(groupdir, str(index) + u'.jpg')file = codecs.open(imagepath, 'wb')file.write(image)file.close()return item

 

转载于:.html