Scrapy Images Downloading

大佬教程收集整理的这篇文章主要介绍了Scrapy Images Downloading，大佬教程大佬觉得挺不错的，现在分享给大家，也给大家做个参考。

如何解决Scrapy Images Downloading？

我的工作最终结果是：

spIDer.py：

import scrapy
import re
import urlparse
from scrapy.spIDers import CrawlSpIDer, Rule
from scrapy.linkextractors import linkExtractor
from scrapy.loader.processors import Join, MapCompose, TakeFirst
from scrapy.pipelines.images import ImagesPipeline
from production.items import ProductionItem
from production.items import ImageItem

class productionSpIDer(scrapy.SpIDer):
    name = "production"
    allowed_domains = ["url"]
    start_urls = [
        "starTingurl.com"
    ]

def parse(self, responsE):
    for sel in response.xpath('//HTML/body'):
        item = ProductionItem()
        img_url = sel.xpath('//a[@IDd="followclaslink"]/@href').extract()[0]
        yIEld scrapy.request(urlparse.urljoin(response.url, img_url),callBACk=self.parseImages,  Meta={'item': item})

def parseImages(self, responsE):
    for elem in response.xpath("//img"):
        img_url = elem.xpath("@src").extract_first()
        yIEld ImageItem(image_urls=[img_url])

SetTings.py

BOt_name = 'production'

SPIDER_MODulES = ['production.spIDers']
NEWSPIDER_MODulE = 'production.spIDers'
DEFAulT_ITEM_CLASS = 'production.items'
ROBOTSTXT_OBEY = True
IMAGES_STORE = '/Users/home/images'

DOWNLOAD_DELAY = 2

ITEM_PIPElines = {'scrapy.pipelines.images.ImagesPipeline': 1}
# disable cookies (enabled by default)
items.py

# -*- Coding: utf-8 -*-
import scrapy

class ProductionItem(scrapy.Item):
    img_url = scrapy.FIEld()
# ScraPingList ResIDential & YIEld Estate for SALE
class ListResIDentialitem(scrapy.Item):
    image_urls = scrapy.FIEld()
    images = scrapy.FIEld()

class ImageItem(scrapy.Item):
    image_urls = scrapy.FIEld()
    images = scrapy.FIEld()

pipelines.py

import scrapy
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem

class MyImagesPipeline(ImagesPipelinE):

    def get_media_requests(self, item, info):
        for image_url in item['image_urls']:
            yIEld scrapy.request(image_url)

    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem("Item contains no images")
        item['image_paths'] = image_paths
        return item

解决方法

import scrapy
import re
import os
import urlparse
from scrapy.spiders import CrawlSpider,Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.loader.processors import Join,MapCompose,TakeFirst
from scrapy.pipelines.images import ImagesPipeline
from production.items import ProductionItem,ListResidentialItem

class productionSpider(scrapy.Spider):
    name = "production"
    allowed_domains = ["someurl.com"]
    start_urls = [
        "someurl.com"
]

def parse(self,responsE):
    for sel in response.xpath('//html/body'):
        item = ProductionItem()
        img_url = sel.xpath('//a[@data-tealium-id="detail_nav_showphotos"]/@href').extract()[0]
        yield scrapy.request(urlparse.urljoin(response.url,img_url),callBACk=self.parseBasicLisTingInfo,meta={'item': item})

def parseBasicLisTingInfo(item,responsE):
    item = response.request.meta['item']
    item = ListResidentialItem()
    try:
        image_urls = map(unicode.Strip,response.xpath('//a[@itemprop="contentUrl"]/@data-href').extract())
        item['image_urls'] = [ x for x in image_urls]
    except IndexError:
        item['image_urls'] = ''

    return item

from scrapy.setTings.default_setTings import ITEM_PIPELInes
from scrapy.pipelines.images import ImagesPipeline

BOt_name = 'production'

SPIDER_MODULES = ['production.spiders']
NEWSPIDER_MODULE = 'production.spiders'
DEFAULT_ITEM_CLASS = 'production.items'

ROBOTSTXT_OBEY = True
DEPTH_PRIORITY = 1
IMAGE_STORE = '/images'

CONCURRENT_requESTS = 250

DOWNLOAD_DELAY = 2

ITEM_PIPELInes = {
    'scrapy.contrib.pipeline.images.ImagesPipeline': 300,}

# -*- coding: utf-8 -*-
import scrapy

class ProductionItem(scrapy.Item):
    img_url = scrapy.Field()

# ScrapingList Residential & Yield Estate for SALE
class ListResidentialItem(scrapy.Item):
    image_urls = scrapy.Field()
    images = scrapy.Field()

    pass

大佬总结

以上是大佬教程为你收集整理的Scrapy Images Downloading全部内容，希望文章能够帮你解决Scrapy Images Downloading所遇到的程序开发问题。

如果觉得大佬教程网站内容还不错，欢迎将大佬教程推荐给程序员好友。

本图文内容来源于网友网络收集整理提供，作为学习参考使用，版权属于原作者。
如您有任何意见或建议可联系处理。小编QQ：384754419，请注明来意。