97 lines
4.7 KiB
Python
97 lines
4.7 KiB
Python
|
"""Offline static HTML quality gate, bounded CPU for multi-thousand-page sites.
|
||
|
|
|
||
|
|
No network requests. Use the standard-library streaming HTML parser and a
|
||
|
|
pre-indexed build manifest instead of repeatedly opening files for every link.
|
||
|
|
"""
|
||
|
|
from html.parser import HTMLParser
|
||
|
|
from pathlib import Path, PurePosixPath
|
||
|
|
import posixpath
|
||
|
|
from urllib.parse import urlsplit,unquote
|
||
|
|
from . import db
|
||
|
|
|
||
|
|
|
||
|
|
class _Scanner(HTMLParser):
|
||
|
|
def __init__(self):
|
||
|
|
super().__init__(convert_charrefs=True)
|
||
|
|
self.title_parts=[];self.in_title=False;self.h1=0;self.description=''
|
||
|
|
self.targets=[];self.missing_alt=[];self.base_root=False
|
||
|
|
|
||
|
|
def handle_starttag(self,tag,attrs):
|
||
|
|
attrs=dict(attrs)
|
||
|
|
if tag=='title':self.in_title=True
|
||
|
|
elif tag=='h1':self.h1+=1
|
||
|
|
elif tag=='base' and attrs.get('href')=='/':self.base_root=True
|
||
|
|
elif tag=='meta' and str(attrs.get('name','')).lower()=='description':
|
||
|
|
self.description=(attrs.get('content') or '').strip()
|
||
|
|
elif tag=='img' and not attrs.get('alt'):
|
||
|
|
self.missing_alt.append((attrs.get('src') or '')[:180])
|
||
|
|
if tag in ('a','img','script','link'):
|
||
|
|
if tag=='link':
|
||
|
|
rel=(attrs.get('rel') or '').lower().split()
|
||
|
|
if 'canonical' in rel or 'alternate' in rel:return
|
||
|
|
target=attrs.get('href' if tag in ('a','link') else 'src')
|
||
|
|
if target:self.targets.append(target)
|
||
|
|
|
||
|
|
def handle_startendtag(self,tag,attrs):self.handle_starttag(tag,attrs)
|
||
|
|
def handle_endtag(self,tag):
|
||
|
|
if tag=='title':self.in_title=False
|
||
|
|
def handle_data(self,data):
|
||
|
|
if self.in_title:self.title_parts.append(data)
|
||
|
|
|
||
|
|
|
||
|
|
def _local_path(target,rel,base_root,hostname):
|
||
|
|
parsed=urlsplit(target)
|
||
|
|
if parsed.scheme in ('http','https'):
|
||
|
|
if parsed.hostname!=hostname:return None
|
||
|
|
path=parsed.path
|
||
|
|
elif parsed.scheme or target.startswith('//') or target.startswith('#'):
|
||
|
|
return None
|
||
|
|
else:path=parsed.path
|
||
|
|
if not path:return None
|
||
|
|
raw=unquote(path)
|
||
|
|
if raw.startswith('/') or base_root:
|
||
|
|
combined=raw.lstrip('/')
|
||
|
|
else:
|
||
|
|
combined=posixpath.join(posixpath.dirname(rel),raw)
|
||
|
|
if combined=='':return 'index.html'
|
||
|
|
normalized=posixpath.normpath(combined)
|
||
|
|
if normalized in ('..','.') or normalized.startswith('../') or '\\' in normalized:
|
||
|
|
return '!unsafe!'
|
||
|
|
if path.endswith('/') or not PurePosixPath(normalized).suffix:
|
||
|
|
normalized=normalized.rstrip('/')+'/index.html'
|
||
|
|
return normalized
|
||
|
|
|
||
|
|
|
||
|
|
def report(site_id,build_id):
|
||
|
|
root=(db.DATA/'builds'/site_id/build_id/'dist').resolve()
|
||
|
|
if not root.is_dir():raise FileNotFoundError('构建版本不存在')
|
||
|
|
# One manifest scan replaces hundreds of thousands of repeated stat calls.
|
||
|
|
indexed={p.relative_to(root).as_posix() for p in root.rglob('*') if p.is_file()}
|
||
|
|
html_files=sorted(x for x in indexed if x.endswith('.html'))
|
||
|
|
site=db.one('SELECT domain FROM sites WHERE id=?',(site_id,)) or {}
|
||
|
|
hostname=urlsplit(site.get('domain') or '').hostname
|
||
|
|
issues=[];seen_titles={};seen_descriptions={}
|
||
|
|
for rel in html_files:
|
||
|
|
scanner=_Scanner()
|
||
|
|
scanner.feed((root/rel).read_text(encoding='utf-8',errors='replace'))
|
||
|
|
scanner.close()
|
||
|
|
title=' '.join(''.join(scanner.title_parts).split())
|
||
|
|
if not title:issues.append({'file':rel,'type':'missing_title','severity':'error'})
|
||
|
|
if scanner.h1!=1:issues.append({'file':rel,'type':'h1_count','severity':'warning','count':scanner.h1})
|
||
|
|
if title and title in seen_titles and Path(rel).name!='404.html':
|
||
|
|
issues.append({'file':rel,'type':'duplicate_title','severity':'warning','other':seen_titles[title]})
|
||
|
|
if title:seen_titles[title]=rel
|
||
|
|
if not scanner.description:issues.append({'file':rel,'type':'missing_description','severity':'warning'})
|
||
|
|
if scanner.description and scanner.description in seen_descriptions:
|
||
|
|
issues.append({'file':rel,'type':'duplicate_description','severity':'warning','other':seen_descriptions[scanner.description]})
|
||
|
|
if scanner.description:seen_descriptions[scanner.description]=rel
|
||
|
|
for img in scanner.missing_alt:
|
||
|
|
issues.append({'file':rel,'type':'missing_image_alt','severity':'warning','src':img})
|
||
|
|
for target in scanner.targets:
|
||
|
|
candidate=_local_path(target,rel,scanner.base_root,hostname)
|
||
|
|
if candidate is not None and candidate not in indexed:
|
||
|
|
issues.append({'file':rel,'type':'broken_internal_reference','severity':'error','target':target[:220]})
|
||
|
|
return {'files':len(html_files),'errors':sum(x['severity']=='error' for x in issues),
|
||
|
|
'warnings':sum(x['severity']=='warning' for x in issues),
|
||
|
|
'passed':not any(x['severity']=='error' for x in issues),'issues':issues[:500]}
|