1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
| from douban_tv import DoubanTVSpider
| from scrapy.crawler import CrawlerProcess
| import sys
|
| if __name__ == "__main__":
| # 获取命令行参数
| type_ = "tv"
| tag = "热门"
| if len(sys.argv) > 1:
| type_ = sys.argv[1]
| if len(sys.argv) > 2:
| tag = sys.argv[2]
|
| process = CrawlerProcess(settings={
| "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/112.0.0.0 Safari/537.36",
| "DOWNLOAD_DELAY": 2,
| "CONCURRENT_REQUESTS": 1,
| "ROBOTSTXT_OBEY": False,
| "LOG_LEVEL": "INFO"
| })
| process.crawl(DoubanTVSpider, type_=type_, tag=tag)
| process.start()
|
|
