bladedragon

pip3 install lxml

pip3 install wheel

pip3 install pyOpenSSL-19.1.0-py2.py3-none-any.whl 

pip3 install Twisted-20.3.0-cp37-cp37m-win_amd64.whl

pip3 install Scrapy

scrapy startproject xxx

import scrapy

#這裡我們需要排名、標題、收藏人數、上映時間和分數
class SpiderDemoItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    index = scrapy.Field()
    title = scrapy.Field()
    star = scrapy.Field()
    releasetime = scrapy.Field()
    score = scrapy.Field()

import scrapy
from spider_demo.items import SpiderDemoItem


class MaoyanSpider(scrapy.Spider):
    #這是爬蟲啟動的名稱，之後啟動爬蟲就需要用到這個名稱
    name = "maoyan"
    #可以爬取的域名可選列表
    allowed_domains = ["maoyan.com"]
    #目標爬取的網址
    start_urls = [
        "http://maoyan.com/board/7/", 
        "http://maoyan.com/board/4/",
    ]

    #處理已經下載的頁面
    def parse(self, response):
        
        dl = response.css(".board-wrapper dd")
        #通過解析得到具體的數據存到容器中
        for dd in dl:
            item = SpiderDemoItem()
            item["index"] = dd.css(".board-index::text").extract_first()
            item["title"] = dd.css(".name a::text").extract_first()
            item["star"] = dd.css(".star::text").extract_first()
            item["releasetime"] = dd.css(".releasetime::text").extract_first()
            score = dd.css('.integer::text').extract_first()
            if score is not None:
                item["score"] = score  + dd.css('.fraction::text').extract_first()
            else:
                item["score"] = 0

            #通過yield將結果返回
            yield item

#如果沒有自動生成UA，就需要手動定義，但是每次爬取都是同樣的UA容易出現驗證操作，因此後面還會介紹一種隨機生成UA的方法
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'

#允許機器人協議，關於機器人協議的具體內容可以自行上網查找
ROBOTSTXT_OBEY = False

import json
import codecs


class SpiderDemoPipeline:
    def process_item(self, item, spider):
        return item


class JsonPipline(object):
    def __init__(self):
        print("打開文件，準備寫入....")
        self.file = codecs.open("maoyan.json", "wb", encoding='utf-8')

    def process_item(self, item, spider):
        print("準備寫入...")
        line = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(line)
        return item

    def close_spider(self, spider):
        print("寫入完畢，關閉文件")
        self.file.close

ITEM_PIPELINES = {
    #    'spider_demo.pipelines.SpiderDemoPipeline': 300,
    'spider_demo.pipelines.JsonPipline': 200
}

from scrapy import signals
import random
from fake_useragent import UserAgent


class RandomUserAgentMiddleware(object):
    # 隨機更換user-agent
    def __init__(self, crawler):
        super(RandomUserAgentMiddleware, self).__init__()
        self.ua = UserAgent()
        self.ua_type = crawler.settings.get("RANDOM_UA_TYPE", "random")

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def process_request(self, request, spider):
        def get_ua():
            return getattr(self.ua, self.ua_type)

        request.headers.setdefault('User-Agent', get_ua())

DOWNLOADER_MIDDLEWARES = {
    'spider_demo.middlewares.SpiderDemoDownloaderMiddleware': 543,
     'spider_demo.middlewares.RandomUserAgentMiddleware': 400,
    'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
}

# 隨機選擇UA
#這個是自己設置的，依賴於fake-useragent
RANDOM_UA_TYPE = 'random'