61 lines
2.8 KiB
Python
61 lines
2.8 KiB
Python
|
"""Dedicated Docker worker: database-backed queue, serialized claim, retry + restart recovery."""
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import signal
|
||
|
|
import time
|
||
|
|
from datetime import datetime, timedelta, timezone
|
||
|
|
from . import db, enterprise, builder
|
||
|
|
|
||
|
|
running=True
|
||
|
|
|
||
|
|
def stop(*_):
|
||
|
|
global running
|
||
|
|
running=False
|
||
|
|
|
||
|
|
def claim():
|
||
|
|
with db.connect() as c:
|
||
|
|
c.execute('BEGIN IMMEDIATE')
|
||
|
|
job=c.execute("""SELECT id FROM build_jobs
|
||
|
|
WHERE state='queued' AND (retry_at IS NULL OR retry_at<=?)
|
||
|
|
ORDER BY created_at,id LIMIT 1""",(db.now(),)).fetchone()
|
||
|
|
if not job:return None
|
||
|
|
c.execute("UPDATE build_jobs SET state='running',attempts=attempts+1,started_at=?,retry_at=NULL WHERE id=?",(db.now(),job['id']))
|
||
|
|
return dict(c.execute('SELECT * FROM build_jobs WHERE id=?',(job['id'],)).fetchone())
|
||
|
|
|
||
|
|
def run_once():
|
||
|
|
job=claim()
|
||
|
|
if job is None:return None
|
||
|
|
try:
|
||
|
|
if job['kind']=='build': result=builder.build(job['site_id'],False)
|
||
|
|
else:raise ValueError('Unsupported job')
|
||
|
|
db.execute("UPDATE build_jobs SET state='done',result_json=?,finished_at=? WHERE id=?",
|
||
|
|
(json.dumps(result,ensure_ascii=False),db.now(),job['id']))
|
||
|
|
except Exception as exc:
|
||
|
|
maximum=max(1,int(os.getenv('SITEFORGE_JOB_MAX_ATTEMPTS','3')))
|
||
|
|
if job['attempts']<maximum:
|
||
|
|
delay=min(300,5*(2**max(job['attempts']-1,0)))
|
||
|
|
retry_at=(datetime.now(timezone.utc)+timedelta(seconds=delay)).isoformat(timespec='seconds')
|
||
|
|
db.execute("""UPDATE build_jobs SET state='queued',error=?,started_at=NULL,
|
||
|
|
finished_at=NULL,retry_at=? WHERE id=?""",
|
||
|
|
(str(exc)[:1200],retry_at,job['id']))
|
||
|
|
else:
|
||
|
|
db.execute("UPDATE build_jobs SET state='failed',error=?,finished_at=?,retry_at=NULL WHERE id=?",
|
||
|
|
(str(exc)[:1200],db.now(),job['id']))
|
||
|
|
return job['id']
|
||
|
|
|
||
|
|
def main():
|
||
|
|
db.init();enterprise.setup()
|
||
|
|
signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop)
|
||
|
|
# A crash before the build record is committed is safe to retry: build IDs
|
||
|
|
# are generated inside builder.build and output directories are immutable.
|
||
|
|
maximum=max(1,int(os.getenv('SITEFORGE_JOB_MAX_ATTEMPTS','3')))
|
||
|
|
db.execute("""UPDATE build_jobs SET state='queued',error='Worker interrupted; retry scheduled',
|
||
|
|
started_at=NULL,finished_at=NULL,retry_at=?
|
||
|
|
WHERE state='running' AND attempts<?""",(db.now(),maximum))
|
||
|
|
db.execute("""UPDATE build_jobs SET state='failed',error='Worker interrupted; retry limit reached',
|
||
|
|
finished_at=?,retry_at=NULL WHERE state='running'""",(db.now(),))
|
||
|
|
while running:
|
||
|
|
if not run_once():time.sleep(float(os.getenv('SITEFORGE_JOB_POLL_SECONDS','1.5')))
|
||
|
|
|
||
|
|
if __name__=='__main__':main()
|