import scrapy
import json
import requests
from datetime import datetime
from urllib.parse import urljoin


class DoubanTVSpider(scrapy.Spider):
    name = 'douban_tv'
    allowed_domains = ['movie.douban.com']

    # 使用start()替代已弃用的start_requests()
    def start_requests(self):
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/112.0.0.0 Safari/537.36',
            'Referer': 'https://movie.douban.com/tv/'
        }
        # 同时爬取电影和电视剧
        # urls = [
        #     "https://movie.douban.com/j/new_search_subjects?sort=U&tag=电视剧&start=0",
        #     "https://movie.douban.com/j/new_search_subjects?sort=U&tag=电影&start=0"
        # ]
        # 路径2最新电视剧
        urls = [
            "https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%83%AD%E9%97%A8&sort=recommend&start=0"
        ]
        for url in urls:
            yield scrapy.Request(url, headers=headers, callback=self.parse)

    def parse(self, response):
        try:
            data = json.loads(response.text)
            self.logger.info(f"解析到的数据: {data}")  # 添加日志
        except json.JSONDecodeError:
            self.logger.error("JSON解析失败: %s", response.url)
            return

        # 检查数据结构
        items = data.get('subjects', [])  # 新API使用subjects字段
        if not items:
            self.logger.error("未找到subjects数据")
            return

        for item in items:
            detail_url = item.get('url')
            if detail_url:
                yield scrapy.Request(
                    detail_url,
                    callback=self.parse_detail,
                    meta={'item': item}
                )

        # 分页逻辑
        current_start = int(response.url.split('start=')[-1])
        next_start = current_start + 20
        if next_start < 60:
            next_url = response.url.replace(f'start={current_start}', f'start={next_start}')
            yield response.follow(next_url, callback=self.parse)

    def parse_detail(self, response):
        item = response.meta['item']

        # 从 JSON 中提取了 url 字段（格式： https://movie.douban.com/subject/25754848/），用于后续请求。
        
        # 获取详细信息
        title = item.get('title', '').strip()
        rate = item.get('rate', '')
        year = item.get('date', '')[:4] if item.get('date') else ''
        
        # 从详情页获取更多信息
        info = response.css('#info').get('')
        
        # 获取导演
        directors = response.css('#info span:contains("导演") + .attrs a::text').getall()
        director = ','.join(directors)

        # 获取演员
        # 使用相邻兄弟选择器定位
        actor = response.css('#info span:contains("主演") + .attrs a::text').getall()
        actors = ','.join(actor)

        # 获取类型
        type_text = ','.join(response.css('#info span[property="v:genre"]::text').getall())
        
        # 获取制片方
        producer = response.xpath('//span[contains(text(), "制片国家/地区")]/following::text()[1]').get().strip()

        # 获取剧情简介
        # 同时提取可见和隐藏内容
        # 提取简介（最终方案）
        synopsises = response.xpath('//span[@property="v:summary"]//text()').getall()
        synopsis =  ' '.join(''.join(synopsises).strip().split())

        # 提取首播年份
        # release_date = response.css('span[property="v:initialReleaseDate"]::text').get()
        # year = release_date.split('-')[0] if release_date else "未知"

        # 提取年份（最优方案）
        year= response.css('.year::text').re_first(r'(\d+)')  # 直接匹配数字

        # 如果正则失效的备选方案
        if not year:
            year_text = response.css('.year::text').get()
            year = year_text.strip('()') if year_text else "未知"

        # 获取封面图片
        cover_url = response.css('#mainpic img::attr(src)').get('')
        cover_alt = response.css('#mainpic img::attr(alt)').get('')

        # 构建API请求数据
        api_data = {
            'nameCn': title,
            'nameEn': '',  # 豆瓣API没有提供英文名
            'type': 'tv',
            'releaseYear': year,
            'director': director,
            'producer': producer,
            'actors': actors,
            'keywords': type_text,
            'synopsis': synopsis,
            'coverUrl': cover_url,
            'coverAlt': cover_alt,
            'userType': 'official'

        }

        # 发送数据到API
        try:
            response = requests.post(
                'http://192.168.1.213:8080/flower/api/filmWorks/new',
                json=api_data,
                headers={'Content-Type': 'application/json'}
            )
            if response.status_code == 200:
                self.logger.info(f"成功保存作品: {title}")
            else:
                self.logger.error(f"保存失败: {title}, 状态码: {response.status_code}")
        except Exception as e:
            self.logger.error(f"API请求失败: {str(e)}")

    # 添加设置（可选）
    custom_settings = {
        'DOWNLOAD_DELAY': 2,  # 增加延迟，避免被封
        'CONCURRENT_REQUESTS': 1,  # 限制并发请求
        'ROBOTSTXT_OBEY': False,
        'LOG_LEVEL': 'DEBUG'  # 设置日志级别为DEBUG以查看更多信息
    }