| | |
|---|
| | | from douban_tv import DoubanTVSpider |
|---|
| | | from scrapy.crawler import CrawlerProcess |
|---|
| | | import sys |
|---|
| | | |
|---|
| | | if __name__ == "__main__": |
|---|
| | | # 获取命令行参数 |
|---|
| | | type_ = "tv" |
|---|
| | | tag = "热门" |
|---|
| | | if len(sys.argv) > 1: |
|---|
| | | type_ = sys.argv[1] |
|---|
| | | if len(sys.argv) > 2: |
|---|
| | | tag = sys.argv[2] |
|---|
| | | |
|---|
| | | process = CrawlerProcess(settings={ |
|---|
| | | "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/112.0.0.0 Safari/537.36", |
|---|
| | | "DOWNLOAD_DELAY": 2, |
|---|
| | |
|---|
| | | "ROBOTSTXT_OBEY": False, |
|---|
| | | "LOG_LEVEL": "INFO" |
|---|
| | | }) |
|---|
| | | process.crawl(DoubanTVSpider) |
|---|
| | | process.crawl(DoubanTVSpider, type_=type_, tag=tag) |
|---|
| | | process.start() |
|---|
