"""SiteForge V8.2: bounded WordPress sitemap discovery, safe media localization
and durable worker-backed REST scanning. All flows require explicit authorization
and remain subject to the existing SiteForge release-review gates.
"""
from __future__ import annotations
import re
import hashlib
import json
import uuid
import zlib
import time
from urllib.parse import urljoin, urlsplit
from bs4 import BeautifulSoup
from defusedxml import ElementTree
from fastapi import APIRouter, Body, HTTPException, Request
from . import db, enterprise, jobs, safe_http, wp_migration_v81 as wp
from .core import route_path

router=APIRouter(prefix='/api/migrations/sites',tags=['V8.2 WordPress migration'])
MAX_SITEMAP_URLS=20_000
MAX_SITEMAP_FILES=60
MAX_MEDIA_BYTES=8_000_000
ALLOWED={'image/png':'.png','image/jpeg':'.jpg','image/webp':'.webp','image/gif':'.gif','image/avif':'.avif'}


def setup():
    with db.connect() as c:
        c.executescript('''
        CREATE TABLE IF NOT EXISTS wp_migration_sitemap(
          session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
          route TEXT NOT NULL, original_url TEXT NOT NULL,
          PRIMARY KEY(session_id,route));
        CREATE TABLE IF NOT EXISTS wp_migration_media(
          session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
          original_url TEXT NOT NULL, local_url TEXT NOT NULL DEFAULT '',
          state TEXT NOT NULL, bytes INTEGER NOT NULL DEFAULT 0,
          error TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL,
          PRIMARY KEY(session_id,original_url));
        ''')


def _site_url(origin,url):
    """Same host and scoped WordPress base directory (no external sitemap)."""
    p=urlsplit(url)
    root=urlsplit(origin)
    if (p.scheme!=root.scheme or p.username or p.password or p.query or p.fragment
        or p.hostname != root.hostname or p.port != root.port):
        raise ValueError('仅允许同一 WordPress 站点的无查询参数 Sitemap 地址')
    prefix=root.path.rstrip('/')+'/'
    if not p.path.startswith(prefix):raise ValueError('Sitemap 超出了来源站点的路径范围')
    return url


def _xml_bytes(response):
    raw=response.content
    if len(raw)>2_000_000:raise ValueError('Sitemap 文件大于 2MB')
    if raw[:2]==b'\x1f\x8b':
        z=zlib.decompressobj(16+zlib.MAX_WBITS)
        raw=z.decompress(raw,3_000_001)
        if len(raw)>3_000_000 or z.unconsumed_tail or not z.eof:
            raise ValueError('Sitemap 压缩包解压后超出限制')
    if len(raw)>3_000_000:raise ValueError('Sitemap 解压后超出限制')
    if b'<!DOCTYPE' in raw.upper() or b'<!ENTITY' in raw.upper():
        raise ValueError('Sitemap 禁止 DTD/实体声明')
    return raw


def discover(site_id,session_id,sitemap_url=None):
    s=wp._one(site_id,session_id)
    if s['state'] in ('cancelled','completed','completed_with_warnings','running'):
        raise HTTPException(409,'当前迁移状态不能修改 Sitemap 清单')
    base=s['origin']
    start=_site_url(base,sitemap_url or urljoin(base,'wp-sitemap.xml'))
    queue=[start];processed=set();urls={}
    while queue:
        if len(processed)>=MAX_SITEMAP_FILES:raise ValueError('Sitemap 文件数超过上限')
        target=queue.pop(0)
        if target in processed:continue
        _site_url(base,target)
        processed.add(target)
        response=safe_http.fetch(target,max_bytes=2_000_000,timeout=15,redirects=0,
                                  headers={'Accept':'application/xml,text/xml'})
        response.raise_for_status()
        root=ElementTree.fromstring(_xml_bytes(response))
        root_name=root.tag.split('}')[-1]
        if root_name not in ('urlset','sitemapindex'):
            raise ValueError('不是有效的 XML Sitemap 或 Sitemap Index')
        for node in root:
            if node.tag.split('}')[-1] not in ('sitemap','url'):continue
            raw=next((v.text.strip() for v in node if v.tag.split('}')[-1]=='loc' and v.text),None)
            if not raw:continue
            if root_name=='sitemapindex':
                queue.append(_site_url(base,raw))
                if len(queue)+len(processed)>MAX_SITEMAP_FILES:raise ValueError('Sitemap Index 数量超过上限')
            else:
                # Different domains and out-of-scope paths cannot enter our baseline.
                _site_url(base,raw)
                urlp=urlsplit(raw)
                normalized=route_path(urlp.path)
                if normalized in urls and urls[normalized]!=raw:raise ValueError('Sitemap 中存在冲突 URL: '+normalized)
                urls[normalized]=raw
                if len(urls)>MAX_SITEMAP_URLS:raise ValueError('Sitemap URL 数量超过上限')
    if not urls:raise ValueError('Sitemap 没有找到任何合法页面 URL')
    with db.connect() as c:
        c.execute('BEGIN IMMEDIATE')
        c.execute('DELETE FROM wp_migration_sitemap WHERE session_id=?',(session_id,))
        c.executemany('INSERT INTO wp_migration_sitemap(session_id,route,original_url) VALUES(?,?,?)',
                      ((session_id,r,u) for r,u in urls.items()))
    return sitemap_status(site_id,session_id)


def sitemap_status(site_id,session_id):
    wp._one(site_id,session_id)
    total=db.one('SELECT COUNT(*) n FROM wp_migration_sitemap WHERE session_id=?',(session_id,))['n']
    missing=db.rows('''SELECT m.route,m.original_url FROM wp_migration_sitemap m
       LEFT JOIN wp_migration_urls u ON u.session_id=m.session_id AND u.route=m.route
       WHERE m.session_id=? AND u.route IS NULL ORDER BY m.route LIMIT 100''',(session_id,))
    missing_count=db.one('''SELECT COUNT(*) n FROM wp_migration_sitemap m LEFT JOIN wp_migration_urls u
       ON u.session_id=m.session_id AND u.route=m.route
       WHERE m.session_id=? AND u.route IS NULL''',(session_id,))['n']
    return {'discovered':total,'missing_count':missing_count,'missing_sample':missing}


def _image_valid(mime,raw):
    if mime=='image/jpeg':return raw.startswith(b'\xff\xd8\xff')
    if mime=='image/png':return raw.startswith(b'\x89PNG\r\n\x1a\n')
    if mime=='image/gif':return raw.startswith((b'GIF87a',b'GIF89a'))
    if mime=='image/webp':return len(raw)>12 and raw[:4]==b'RIFF' and raw[8:12]==b'WEBP'
    if mime=='image/avif':return len(raw)>12 and raw[4:8]==b'ftyp' and b'avif' in raw[8:32]
    return False


def _download_image(origin,url,site_id,session_id):
    p=urlsplit(url);base=urlsplit(origin)
    # Explicitly don't follow offsite CDNs or credential-bearing URLs automatically.
    if p.scheme not in ('http','https') or p.hostname!=base.hostname or p.port!=base.port or p.username or p.password:
        raise ValueError('外部 CDN 或跨域资源需要单独授权')
    r=safe_http.fetch(url,max_bytes=MAX_MEDIA_BYTES,timeout=15,redirects=0)
    r.raise_for_status()
    mime=r.headers.get('content-type','').split(';',1)[0].lower().strip()
    if mime not in ALLOWED or not _image_valid(mime,r.content):raise ValueError('不支持或无效的图片类型')
    # Store only simple flat images; never use remote paths as local filenames.
    name='wp-'+hashlib.sha256(url.encode()).hexdigest()[:24]+ALLOWED[mime]
    target=db.DATA/'media'/site_id/name
    target.parent.mkdir(parents=True,exist_ok=True)
    if not target.exists():
        temp=target.with_suffix(target.suffix+'.'+uuid.uuid4().hex[:8]+'.tmp')
        try:temp.write_bytes(r.content);temp.replace(target)
        finally:temp.unlink(missing_ok=True)
    return '/media/'+name,len(r.content)


def _srcset_urls(value):
    """Extract standard WordPress srcset URL candidates, preserving width/density hints."""
    if not isinstance(value,str):return []
    result=[]
    for candidate in value.split(','):
        fields=candidate.strip().split()
        if fields and fields[0].startswith(('http://','https://','//')):
            result.append(fields[0])
    return result


# CSS background images are common in Elementor containers and ACF galleries.
# Restrict to syntactically explicit remote URLs, never evaluate arbitrary CSS.
CSS_URL = re.compile(r'''url\(\s*([\"\']?)((?:https?:)?//[^\"\')\s]+)\1\s*\)''', re.I)


def _css_urls(raw):
    return [match.group(2) for match in CSS_URL.finditer(str(raw or ''))]


def _rewrite_css_urls(raw,converted):
    def replace(match):
        url=match.group(2)
        if url not in converted:return match.group(0)
        quote=match.group(1)
        return f'url({quote}{converted[url]}{quote})'
    updated=CSS_URL.sub(replace,str(raw or ''))
    return updated,updated!=str(raw or '')


def _media_urls(content):
    """Collect images plus responsive/lazy/CSS backgrounds for explicit review."""
    urls=[content.get('cover') or '']
    soup=BeautifulSoup(content.get('body') or '','html.parser')
    for tag in soup.find_all(True):
        for key in ('src','data-src','poster','data-background','data-background-image'):
            if tag.get(key):urls.append(tag[key])
        for key in ('srcset','data-srcset'):
            urls.extend(_srcset_urls(tag.get(key)))
        if tag.get('style'):urls.extend(_css_urls(tag['style']))
        if tag.name=='style':urls.extend(_css_urls(tag.get_text()))
    return list(dict.fromkeys(u for u in urls if isinstance(u,str)
                              and u.startswith(('https://','http://','//'))))


def _rewrite_srcset(value,converted):
    candidates=[];changed=False
    for part in str(value).split(','):
        fields=part.strip().split()
        if not fields:continue
        url=fields[0]
        if url in converted:
            fields[0]=converted[url];changed=True
        candidates.append(' '.join(fields))
    return ', '.join(candidates),changed


def _rewrite_media_html(body,converted):
    soup=BeautifulSoup(body,'html.parser');changed=False
    for tag in soup.find_all(True):
        for key in ('src','data-src','poster','data-background','data-background-image'):
            if tag.get(key) in converted:
                tag[key]=converted[tag[key]];changed=True
        for key in ('srcset','data-srcset'):
            if not tag.get(key):continue
            new,updated=_rewrite_srcset(tag[key],converted)
            if updated:
                tag[key]=new;changed=True
                if tag.name=='img' and key=='srcset' and len(new.split(','))==1:
                    fields=new.strip().split()
                    if fields and fields[0]==tag.get('src'):
                        tag.attrs.pop(key,None)
        if tag.get('style'):
            value,updated=_rewrite_css_urls(tag['style'],converted)
            if updated:tag['style']=value;changed=True
        if tag.name=='style':
            value,updated=_rewrite_css_urls(tag.string if tag.string is not None else tag.get_text(),converted)
            if updated:tag.string=value;changed=True
    return str(soup) if changed else body,changed


def localize_media(site_id,session_id,limit=20):
    s=wp._one(site_id,session_id)
    if not 1<=int(limit)<=40:raise ValueError('单次处理媒体数量须为 1–40')
    if s['state'] in ('running','cancelled','completed','completed_with_warnings'):
        raise HTTPException(409,'请先暂停采集，且必须在确认导入前执行媒体本地化')
    batches=db.rows('''SELECT b.id FROM wp_migration_pages p JOIN v8_import_batches b ON b.id=p.batch_id
       WHERE p.session_id=? AND b.site_id=? AND b.state='staged' ORDER BY p.phase,p.page''',(session_id,site_id))
    items=[]
    for b in batches:
        items.extend(db.rows('SELECT id,content_json FROM v8_import_items WHERE batch_id=? ORDER BY row_number',(b['id'],)))
    unique=list(dict.fromkeys(url for item in items
      for url in _media_urls(json.loads(item['content_json']))))
    if len(unique)>4000:raise ValueError('媒体引用数量超过单次迁移安全上限 4000')
    failures=[];done=0
    for url in unique:
        existing=db.one('SELECT * FROM wp_migration_media WHERE session_id=? AND original_url=?',(session_id,url))
        if existing and existing['state']=='localized':continue
        if done>=int(limit):break
        done+=1
        try:
            local,size=_download_image(s['origin'],url,site_id,session_id)
            state='localized';error=''
        except Exception as exc:
            local='';size=0;state='review_required';error=str(exc)[:240]
            failures.append({'url':url,'reason':error})
        db.execute('''INSERT INTO wp_migration_media(session_id,original_url,local_url,state,bytes,error,updated_at)
          VALUES(?,?,?,?,?,?,?) ON CONFLICT(session_id,original_url) DO UPDATE SET
          local_url=excluded.local_url,state=excluded.state,bytes=excluded.bytes,
          error=excluded.error,updated_at=excluded.updated_at''',
          (session_id,url,local,state,size,error,db.now()))
    # Rewrite staged content only, never an already-committed or hand-edited entry.
    converted={r['original_url']:r['local_url'] for r in db.rows('SELECT * FROM wp_migration_media WHERE session_id=? AND state=?',(session_id,'localized'))}
    with db.connect() as c:
        c.execute('BEGIN IMMEDIATE')
        for item in items:
            obj=json.loads(item['content_json']);modified=False
            if obj.get('cover') in converted:obj['cover']=converted[obj['cover']];modified=True
            if obj.get('body'):
                updated,replaced=_rewrite_media_html(obj['body'],converted)
                if replaced:obj['body']=updated;modified=True
            if modified:c.execute('UPDATE v8_import_items SET content_json=? WHERE id=?',(json.dumps(obj,ensure_ascii=False),item['id']))
    return {'attempted':done,'localized':len(converted),'failed':failures,'total_candidates':len(unique),
            'remaining':len([x for x in unique if x not in converted]),'requires_review':bool(failures)}


def _findings(site_id,session_id):
    # Sitemap-only URLs must block claims of complete migration.
    sitemap=sitemap_status(site_id,session_id)
    issues=[f'站点地图中未由 REST 采集的页面：{r["route"]}' for r in sitemap['missing_sample']]
    if sitemap['missing_count']>len(sitemap['missing_sample']):
        issues.append(f'其他未采集 Sitemap URL 数量：{sitemap["missing_count"]-len(sitemap["missing_sample"])}')
    for r in db.rows('SELECT original_url,error FROM wp_migration_media WHERE session_id=? AND state!=?',
                     (session_id,'localized')):
        issues.append('媒体未本地化：'+r['original_url'][:170]+'；'+r['error'][:90])
    for b in db.rows('''SELECT p.batch_id FROM wp_migration_pages p JOIN v8_import_batches b ON b.id=p.batch_id
       WHERE p.session_id=? AND b.site_id=?''',(session_id,site_id)):
        if not b['batch_id']:continue
        for record in db.rows('SELECT route,content_json FROM v8_import_items WHERE batch_id=?',(b['batch_id'],)):
            content=json.loads(record['content_json'])
            urls=_media_urls(content)
            if urls:
                issues.append('正文或封面仍引用外部图片：'+record['route'])
    return list(dict.fromkeys(issues))[:500]


def submit_scan(site_id,session_id,actor,key=None):
    s=wp._one(site_id,session_id)
    if s['state'] in ('ready','cancelled','completed','completed_with_warnings','limit_reached'):
        raise HTTPException(409,'迁移采集已经完成或被取消')
    return jobs.submit_migration(site_id,session_id,actor,key)


def run_scan(site_id,session_id,job_id,max_pages=500):
    """Run by isolated Worker subprocess only; do not auto-commit staged data."""
    count=0
    while count<max_pages:
        job=db.one('SELECT cancel_requested FROM build_jobs WHERE id=? AND site_id=?',(job_id,site_id))
        if job is None or job['cancel_requested']:return {'state':'cancelled','pages_processed':count}
        s=wp._one(site_id,session_id)
        if s['state']=='running' and s['lease_until'] and s['lease_until']>db.now():
            # Previous worker may have died while fetching; wait for its per-page
            # lease instead of burning all task retries during the 60-second lease.
            waited=0
            while waited<70 and wp._one(site_id,session_id)['lease_until'] and wp._one(site_id,session_id)['lease_until']>db.now():
                row=db.one('SELECT cancel_requested FROM build_jobs WHERE id=?',(job_id,))
                if not row or row['cancel_requested']:return {'state':'cancelled','pages_processed':count}
                time.sleep(1);waited+=1
            if waited>=70:raise TimeoutError('迁移页锁未释放，等待后再重试')
        if s['state'] in ('ready','cancelled','limit_reached','completed','completed_with_warnings'):
            return {'state':s['state'],'pages_processed':count,'fetched':s['fetched']}
        wp.advance(site_id,session_id)
        count+=1
    return {'state':'page_limit','pages_processed':count}


@router.post('/{site_id}/wordpress/sessions/{session_id}/sitemap')
def api_sitemap(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
    enterprise.require(request,site_id,'editor')
    try:return discover(site_id,session_id,payload.get('url'))
    except (ValueError,safe_http.OutboundError,ElementTree.ParseError) as exc:raise HTTPException(400,str(exc)) from exc

@router.get('/{site_id}/wordpress/sessions/{session_id}/sitemap')
def api_sitemap_status(request:Request,site_id:str,session_id:str):
    enterprise.require(request,site_id,'viewer')
    return sitemap_status(site_id,session_id)

@router.post('/{site_id}/wordpress/sessions/{session_id}/media/localize')
def api_media(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
    enterprise.require(request,site_id,'editor')
    try:return localize_media(site_id,session_id,int(payload.get('limit',20)))
    except (ValueError,TypeError) as exc:raise HTTPException(400,str(exc)) from exc

@router.post('/{site_id}/wordpress/sessions/{session_id}/scan')
def api_scan(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
    enterprise.require(request,site_id,'editor')
    try:return submit_scan(site_id,session_id,request.state.actor,payload.get('idempotency_key'))
    except (jobs.JobConflict,jobs.JobCapacityError) as exc:raise HTTPException(409,str(exc)) from exc