| 12345678910111213141516171819202122232425262728293031323334353637383940414243 |
- """
- Scrapy 通用爬虫框架
- 提供基础的爬虫类、中间件和 Pipeline
- """
- import scrapy
- from scrapy.crawler import CrawlerProcess
- from scrapy.utils.project import get_project_settings
- class BasePharmSpider(scrapy.Spider):
- custom_settings = {
- "DOWNLOAD_DELAY": 3,
- "RANDOMIZE_DOWNLOAD_DELAY": True,
- "CONCURRENT_REQUESTS": 4,
- "CONCURRENT_REQUESTS_PER_DOMAIN": 2,
- "DOWNLOAD_TIMEOUT": 30,
- "RETRY_TIMES": 3,
- "RETRY_HTTP_CODES": [500, 502, 503, 504, 408, 429],
- "USER_AGENT": (
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
- "AppleWebKit/537.36 (KHTML, like Gecko) "
- "Chrome/120.0.0.0 Safari/537.36"
- ),
- "DEFAULT_REQUEST_HEADERS": {
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
- "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
- },
- "ROBOTSTXT_OBEY": True,
- "DOWNLOADER_MIDDLEWARES": {
- "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
- "crawlers.middleware.RotateUserAgentMiddleware": 400,
- "crawlers.middleware.ProxyMiddleware": 410,
- },
- }
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.ua_pool = [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/17.1",
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/119.0.0.0",
- ]
|
