update
This commit is contained in:
team3
670
backend/guide.py
670
backend/guide.py
@@ -24,7 +24,8 @@ from config import (
|
||||
READABILITY_ACTIVE, TEMPLATES_DIR,
|
||||
)
|
||||
import readability
|
||||
from database import list_guides, update_guide, list_blocks, list_subblocks, set_guide_content, get_guide_content, get_outline
|
||||
from database import (list_guides, update_guide, list_blocks, list_subblocks, set_guide_content,
|
||||
get_guide_content, get_outline, guide_stage_counts, delete_guide_board)
|
||||
from fsutil import atomic_write_json, atomic_write_text
|
||||
from jsonio import read_json_file as _json_file, parse_json_text as _parse_json_text
|
||||
from paths import blocks_path, guide_content_path, project_dir, subblocks_path
|
||||
@@ -41,20 +42,16 @@ from textkit import (
|
||||
|
||||
log = logging.getLogger("creator.guide")
|
||||
|
||||
GUIDE_STEPS = ("Outline", "Content", "Content-Check", "Writing", "Reading-Exam")
|
||||
|
||||
# Content/Content-Check/Reading-Exam run in packets of ~GUIDE_CHUNK blocks per agent.
|
||||
# Only the writer (Writing) stays at 1 agent per block (variable lengths, no trimming, no
|
||||
# length alignment between blocks).
|
||||
GUIDE_CHUNK = 10
|
||||
|
||||
# Check steps as a panel: CHECK_PANEL judges per chunk, section flagged on a majority.
|
||||
# A single judge is bias/sampling prone; a small panel is more stable.
|
||||
CHECK_PANEL = 3
|
||||
|
||||
# Reading exam: only ONE round (Check + Fix). Follow-up rounds added little value
|
||||
# (1 agent per block checks finely anyway) but cost extra agents.
|
||||
READING_ROUNDS = 1
|
||||
|
||||
|
||||
# Valid level values: new (learning path) + old (difficulty) backward-compatible.
|
||||
@@ -93,26 +90,8 @@ def _level_label(s: dict) -> str:
|
||||
return "peripheral" if s.get("relevance") == "peripheral" else (s.get("level") or "beginner")
|
||||
|
||||
|
||||
def _assignment_subs(chunk: list[dict], entries: dict[int, str], subs_by_title: dict[str, list[dict]]) -> str:
|
||||
"""Lists the blocks per chapter, with their subblocks and level labels beneath."""
|
||||
lines: list[str] = []
|
||||
for ch in chunk:
|
||||
lines.append(f"CHAPTER: {ch['title']}")
|
||||
for num in ch["nums"]:
|
||||
lines.append(f"- {entries[num]}")
|
||||
for s in subs_by_title.get(_title(entries[num]), []):
|
||||
lines.append(f" [{_level_label(s)}] {s['title']}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _guide_files(content_path: Path) -> dict:
|
||||
d, stem = content_path.parent, content_path.stem
|
||||
return {
|
||||
"outline_slots": [d / f"{stem}.outline-{i}.json" for i in (1, 2, 3)],
|
||||
"outline": d / f"{stem}.outline.json", # judge output
|
||||
# chunk/reading-check/fix files are dynamic:
|
||||
# {stem}.chunk-i.md, {stem}.lese-check-r{n}-{i}.json, {stem}.fix-r{n}-{i}.md
|
||||
}
|
||||
|
||||
|
||||
def guide_slot_files(content_path: Path) -> list[Path]:
|
||||
@@ -120,132 +99,23 @@ def guide_slot_files(content_path: Path) -> list[Path]:
|
||||
return [p for p in content_path.parent.glob(f"{content_path.stem}.*") if p != content_path]
|
||||
|
||||
|
||||
def _done_path(content_path: Path) -> Path:
|
||||
return content_path.parent / f"{content_path.stem}.done"
|
||||
|
||||
|
||||
def guide_done_step(content_path: Path) -> int:
|
||||
"""Highest FULLY completed step index (marker per topic+format). -1 = none.
|
||||
If the content file exists, all steps are done."""
|
||||
if content_path.exists():
|
||||
return len(GUIDE_STEPS) - 1
|
||||
try:
|
||||
return int(_done_path(content_path).read_text(encoding="utf-8").strip())
|
||||
except (OSError, ValueError):
|
||||
return -1
|
||||
|
||||
|
||||
def _set_done(content_path: Path, step: int) -> None:
|
||||
"""Set marker to `step` — monotone (only increase), except on the re-run reset (force)."""
|
||||
if step > guide_done_step(content_path):
|
||||
atomic_write_text(_done_path(content_path), str(step))
|
||||
|
||||
|
||||
def _reset_done(content_path: Path, step: int) -> None:
|
||||
"""Set marker hard to `step` (for re-run from step; step may decrease)."""
|
||||
if step < 0:
|
||||
_done_path(content_path).unlink(missing_ok=True)
|
||||
else:
|
||||
atomic_write_text(_done_path(content_path), str(step))
|
||||
|
||||
|
||||
# Slot-file globs per step (index = GUIDE_STEPS). Stem-anchored, collision-free.
|
||||
_STEP_GLOBS = (
|
||||
("outline*",), # 0 Outline (incl. selection filter)
|
||||
("content-chunk-*", "content-nach-*"), # 1 Content (incl. follow-up round)
|
||||
("content-check-*", "content-fix-*"), # 2 Content-Check
|
||||
("chunk-*",), # 3 Writing (chunk-* also matches chunk-nach-*)
|
||||
("lese-check-*", "fix-r*"), # 4 Reading-Exam
|
||||
)
|
||||
|
||||
|
||||
def _reset_guide_from_step(content_path: Path, step: int) -> None:
|
||||
"""Re-run from step: delete content + all slot files of steps ≥ step.
|
||||
Earlier steps stay → the resume rebuilds from `step` (everything below is reused)."""
|
||||
content_path.unlink(missing_ok=True) # no longer "done" → no fresh-start wipe
|
||||
d, stem = content_path.parent, content_path.stem
|
||||
for globs in _STEP_GLOBS[step:]:
|
||||
for pat in globs:
|
||||
for p in d.glob(f"{stem}.{pat}"):
|
||||
p.unlink(missing_ok=True)
|
||||
_reset_done(content_path, step - 1) # steps < step count as done
|
||||
|
||||
|
||||
def _read_problems_schema(data):
|
||||
"""{"ok": true} → [] · {"problems": [{"section", "problem"}]} → list · else None."""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
if data.get("ok") is True:
|
||||
return []
|
||||
p = data.get("problems")
|
||||
if not isinstance(p, list) or not p:
|
||||
return None
|
||||
out = []
|
||||
for x in p:
|
||||
if not isinstance(x, dict) or not isinstance(x.get("section"), str) or not isinstance(x.get("problem"), str):
|
||||
return None
|
||||
out.append({"section": x["section"].strip(), "problem": x["problem"].strip()})
|
||||
return out or None
|
||||
|
||||
|
||||
def _panel_problems(judge_paths: list[Path], valid: set[int], idx: dict[str, int]) -> dict[int, str]:
|
||||
"""Panel aggregation: several judge outputs of a chunk → flagged {num: problem}.
|
||||
|
||||
One vote per judge that names a section. Flagged when more than half of the
|
||||
DELIVERED (validly parsed) judges name it (3→≥2, 2→≥2, 1→≥1). Robust against a
|
||||
single failure: missing files do not count. Problem text from the first naming judge.
|
||||
"""
|
||||
outputs = [p for p in (_read_problems_schema(_json_file(j)) for j in judge_paths) if p is not None]
|
||||
if not outputs:
|
||||
return {}
|
||||
votes: dict[int, int] = {}
|
||||
problem: dict[int, str] = {}
|
||||
for out in outputs:
|
||||
seen: set[int] = set()
|
||||
for item in out:
|
||||
num = _resolve_title(idx, item["section"])
|
||||
if num is None or num not in valid or num in seen:
|
||||
continue
|
||||
seen.add(num)
|
||||
votes[num] = votes.get(num, 0) + 1
|
||||
problem.setdefault(num, item["problem"])
|
||||
threshold = len(outputs) / 2
|
||||
return {num: problem[num] for num, v in votes.items() if v > threshold}
|
||||
|
||||
|
||||
def _resolve_outline(data, entries: dict[int, str], target_min: int, target_max: int) -> list[dict] | None:
|
||||
"""{"chapters": [{"title", "numbers": [1, 3, 7]}]} → [{"title", "nums"}].
|
||||
|
||||
Numbers are the IDs from `entries` (1-based, as presented to the agent).
|
||||
`target_min`/`target_max` = allowed range of selected blocks (with a small tolerance).
|
||||
"""
|
||||
if not isinstance(data, dict) or not isinstance(data.get("chapters"), list):
|
||||
return None
|
||||
valid = set(entries)
|
||||
chapters: list[dict] = []
|
||||
seen: set[int] = set()
|
||||
total = unknown = 0
|
||||
for ch in data["chapters"]:
|
||||
if not isinstance(ch, dict) or not isinstance(ch.get("numbers"), list):
|
||||
return None
|
||||
nums = []
|
||||
for t in ch["numbers"]:
|
||||
total += 1
|
||||
num = t if isinstance(t, int) and not isinstance(t, bool) else None
|
||||
if num is None or num not in valid:
|
||||
unknown += 1
|
||||
elif num not in seen:
|
||||
nums.append(num)
|
||||
seen.add(num)
|
||||
if nums:
|
||||
chapters.append({"title": str(ch.get("title", "")).strip() or "Chapter", "nums": nums})
|
||||
if not chapters or total == 0:
|
||||
return None
|
||||
if (total - unknown) / total < 0.85:
|
||||
return None
|
||||
if len(seen) < 0.9 * target_min or len(seen) > 1.1 * target_max:
|
||||
return None
|
||||
return chapters
|
||||
|
||||
|
||||
def _fallback_outline(entries: dict[int, str]) -> list[dict]:
|
||||
@@ -324,515 +194,6 @@ async def _outline_from_db(topic: str, sel_entries: dict[int, str]) -> list[dict
|
||||
return plan or None
|
||||
|
||||
|
||||
async def _generate_sections(
|
||||
guide_id: str, topic: str, format_name: str, entries: dict[int, str],
|
||||
facts: str, instructions: str, provider: str,
|
||||
content_path: Path,
|
||||
) -> list[dict] | None:
|
||||
def is_cancelled() -> bool:
|
||||
return is_guide_cancelled(guide_id)
|
||||
|
||||
ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled, guide_id=guide_id)
|
||||
spec = (TEMPLATES_DIR / "Format" / "Section.md").read_text(encoding="utf-8")
|
||||
files = _guide_files(content_path)
|
||||
zweck = FORMAT_PURPOSE[format_name]
|
||||
|
||||
# Subblocks per block (DB-first) — loaded early: drives selection + sub-filter per format.
|
||||
# Missing → {} (fallback: guide takes everything).
|
||||
subs_raw = await _load_subblocks(topic)
|
||||
# Extract-once grounding: stored, verified facts replace the generic source hint.
|
||||
# The content agent phrases from them instead of reading the source again.
|
||||
if (facts_block := _facts_grounding(subs_raw)):
|
||||
facts = facts_block
|
||||
|
||||
def _has_relevance(num, kind):
|
||||
return any(isinstance(s, dict) and s.get("relevance") == kind for s in subs_raw.get(_title(entries[num]), []))
|
||||
|
||||
# Selection: ONE full document with ALL blocks (incl. peripheral). The views E/M/S/F
|
||||
# filter later per subblock level. (FullGuide/Rest remain as legacy branches.)
|
||||
if format_name == "Rest":
|
||||
selection = [num for num in entries if not _has_relevance(num, "relevant")]
|
||||
else: # Guide / FullGuide → all blocks
|
||||
selection = list(entries)
|
||||
if not selection:
|
||||
await _fail(guide_id, "No matching blocks for this format")
|
||||
return None
|
||||
|
||||
sel_entries = {num: entries[num] for num in selection}
|
||||
target = len(sel_entries)
|
||||
# Numbered list (ID = block number from entries) — agents/judge order by number.
|
||||
sel_list = "\n".join(f"{num}. {t}" for num, t in sel_entries.items())
|
||||
|
||||
# Step 0: outline. Prefers the blocks artifact (DB) — the guide only presents,
|
||||
# no longer structures itself. Missing (legacy) → previous agents/judge logic as fallback.
|
||||
# 0 valid → code fallback, 1 → direct, ≥2 → judge (with proposal as fallback).
|
||||
plan = await _outline_from_db(topic, sel_entries)
|
||||
if plan is not None:
|
||||
_log(topic, f"Outline from blocks artifact ({len(plan)} chapters)")
|
||||
if plan is None:
|
||||
plan = _resolve_outline(_json_file(files["outline"]), sel_entries, target, target)
|
||||
if plan is None:
|
||||
await _set_step(guide_id, 0, "Outline proposals (3 agents)…")
|
||||
files["outline"].unlink(missing_ok=True)
|
||||
proposals: list[list[dict]] = []
|
||||
pending = []
|
||||
for i, path in enumerate(files["outline_slots"], 1):
|
||||
res = _resolve_outline(_json_file(path), sel_entries, target, target)
|
||||
if res is not None:
|
||||
proposals.append(res)
|
||||
else:
|
||||
pending.append((i, path))
|
||||
if len(proposals) < 3 and pending:
|
||||
slots = [
|
||||
{
|
||||
"key": f"{guide_id}-outline-{i}",
|
||||
"prompt": _prompt(
|
||||
"Guide-Outline",
|
||||
topic=topic, format_name=format_name, blocks=sel_list,
|
||||
out_path=path, extra=_extra(instructions),
|
||||
),
|
||||
"role": "guide", "capabilities": "files",
|
||||
"payload": (lambda result, p=path: _resolve_outline(_json_file(p), sel_entries, target, target)),
|
||||
}
|
||||
for i, path in pending
|
||||
]
|
||||
# Quorum 1: take whatever comes — no minimum requirement, no abort.
|
||||
new = await _race(
|
||||
topic, "Outline", slots, 1, _timeout("plan", target),
|
||||
provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE,
|
||||
)
|
||||
if is_cancelled():
|
||||
return None
|
||||
proposals += new or []
|
||||
|
||||
if not proposals:
|
||||
_log(topic, "Outline: no valid proposal — deterministic fallback")
|
||||
plan = _fallback_outline(sel_entries)
|
||||
elif len(proposals) == 1:
|
||||
plan = proposals[0] # one proposal → no judge needed
|
||||
else:
|
||||
await _set_step(guide_id, 0, "Merging outlines…")
|
||||
proposals_text = "\n\n".join(
|
||||
f"### Proposal {i}\n"
|
||||
+ "\n".join(f"CHAPTER: {ch['title']}\n Numbers: {', '.join(str(num) for num in ch['nums'])}" for ch in v)
|
||||
for i, v in enumerate(proposals, 1)
|
||||
)
|
||||
status, plan = await run_single_slot(
|
||||
ctx, "Outline-Judge",
|
||||
key=f"{guide_id}-outline-judge",
|
||||
prompt=_prompt(
|
||||
"Guide-Outline-Judge",
|
||||
topic=topic, format_name=format_name, purpose=zweck, n=len(proposals),
|
||||
blocks=sel_list, outlines=proposals_text,
|
||||
out_path=files["outline"], extra=_extra(instructions),
|
||||
),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result: _resolve_outline(_json_file(files["outline"]), sel_entries, target, target),
|
||||
timeout=_timeout("plan_judge", target),
|
||||
)
|
||||
if status == CANCELLED:
|
||||
return None
|
||||
if status == FAILED or plan is None:
|
||||
_log(topic, "Outline judge produced no result — best proposal kept")
|
||||
plan = proposals[0]
|
||||
|
||||
# Guarantee: every selected block is in the plan (against dropping agents/judges).
|
||||
plan = _with_remainder(plan, sel_entries)
|
||||
_set_done(content_path, 0) # outline ready
|
||||
|
||||
# Coarse chunks (~GUIDE_CHUNK blocks per agent) for content, content-check and reading-exam.
|
||||
# The writer builds per block beneath (its own fine chunks, see below) → variable lengths.
|
||||
total_sections = sum(len(c["nums"]) for c in plan)
|
||||
chunks = _split_chunks(plan, max(1, math.ceil(total_sections / GUIDE_CHUNK)))
|
||||
# Subblocks per block: Guide/FullGuide take ALL (incl. peripheral → level 4 in the view);
|
||||
# only the legacy Rest branch filters to peripheral. So the one document carries all levels.
|
||||
if format_name == "Rest":
|
||||
subs_by_title = {t: [s for s in subs if s.get("relevance") == "peripheral"] for t, subs in subs_raw.items()}
|
||||
else: # Guide / FullGuide
|
||||
subs_by_title = {t: list(subs) for t, subs in subs_raw.items()}
|
||||
subs_by_title = {t: subs for t, subs in subs_by_title.items() if subs}
|
||||
assignments = [_assignment_subs(chunk, entries, subs_by_title) for chunk in chunks]
|
||||
chunk_sizes = [sum(len(c["nums"]) for c in chunk) for chunk in chunks]
|
||||
writer_count = len(chunks)
|
||||
idx = _title_index(entries)
|
||||
|
||||
# Step 2: identify content per block — one agent per chunk (marker output, resume).
|
||||
content_paths = [content_path.parent / f"{content_path.stem}.content-chunk-{i}.md" for i in range(1, writer_count + 1)]
|
||||
pending = [i for i, p in enumerate(content_paths) if not p.exists()]
|
||||
if pending:
|
||||
async def report(d, t): await _set_step(guide_id, 1, f"Gathering content {d}/{t}…")
|
||||
results = await _gather_progress([
|
||||
run_agent(
|
||||
f"{guide_id}-content-{i + 1}",
|
||||
_prompt(
|
||||
"Guide-Content",
|
||||
topic=topic, assignment=assignments[i], facts=facts,
|
||||
out_path=content_paths[i], extra=_extra(instructions),
|
||||
),
|
||||
_timeout("content", chunk_sizes[i]), provider=provider, role="guide", capabilities="full",
|
||||
)
|
||||
for i in pending
|
||||
], writer_count, report, start=writer_count - len(pending))
|
||||
if is_cancelled():
|
||||
return None
|
||||
if not any(p.exists() for p in content_paths):
|
||||
await _fail(guide_id, _gather_error("Content error", list(results)))
|
||||
return None
|
||||
|
||||
content_by_num: dict[int, str] = {}
|
||||
for p in content_paths:
|
||||
if not p.exists():
|
||||
continue
|
||||
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
|
||||
num = _resolve_title(idx, sec["title"])
|
||||
if num is not None and num not in content_by_num and sec["md"].strip():
|
||||
content_by_num[num] = sec["md"]
|
||||
if not content_by_num:
|
||||
await _fail(guide_id, "No content identified")
|
||||
return None
|
||||
|
||||
# Follow-up round: pull missing blocks (chunk failure or lazy output) deliberately — one round.
|
||||
planned_nums = [num for ch in plan for num in ch["nums"]]
|
||||
missing = [num for num in planned_nums if num not in content_by_num]
|
||||
if missing:
|
||||
_log(topic, f"Content: {len(missing)} block(s) missing — follow-up round…")
|
||||
followup_chunks = [[{"title": "Additional", "nums": missing[k:k + GUIDE_CHUNK]}] for k in range(0, len(missing), GUIDE_CHUNK)]
|
||||
followup_paths = [content_path.parent / f"{content_path.stem}.content-nach-{k}.md" for k in range(1, len(followup_chunks) + 1)]
|
||||
followup_pending = [k for k, p in enumerate(followup_paths) if not p.exists()]
|
||||
if followup_pending:
|
||||
async def report_n(d, t): await _set_step(guide_id, 1, f"Gathering missing content {d}/{t}…")
|
||||
await _gather_progress([
|
||||
run_agent(
|
||||
f"{guide_id}-content-nach-{k + 1}",
|
||||
_prompt(
|
||||
"Guide-Content",
|
||||
topic=topic, assignment=_assignment_subs(followup_chunks[k], entries, subs_by_title),
|
||||
facts=facts, out_path=followup_paths[k], extra=_extra(instructions),
|
||||
),
|
||||
_timeout("content", len(followup_chunks[k][0]["nums"])), provider=provider, role="guide", capabilities="full",
|
||||
)
|
||||
for k in followup_pending
|
||||
], len(followup_chunks), report_n, start=len(followup_chunks) - len(followup_pending))
|
||||
if is_cancelled():
|
||||
return None
|
||||
for p in followup_paths:
|
||||
if not p.exists():
|
||||
continue
|
||||
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
|
||||
num = _resolve_title(idx, sec["title"])
|
||||
if num is not None and num not in content_by_num and sec["md"].strip():
|
||||
content_by_num[num] = sec["md"]
|
||||
|
||||
if all(p.exists() for p in content_paths):
|
||||
_set_done(content_path, 1) # content complete
|
||||
|
||||
content_chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in content_by_num] for chunk in chunks]
|
||||
|
||||
# Step 3: check content — CHECK_PANEL judges per chunk, majority flags.
|
||||
# + revise flagged ones once. Resume: only restart missing judge files.
|
||||
check_judge_paths = [
|
||||
[content_path.parent / f"{content_path.stem}.content-check-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)]
|
||||
for i in range(1, writer_count + 1)
|
||||
]
|
||||
pending_slots = [
|
||||
(i, j) for i in range(writer_count) if content_chunk_nums[i]
|
||||
for j in range(CHECK_PANEL) if _read_problems_schema(_json_file(check_judge_paths[i][j])) is None
|
||||
]
|
||||
if pending_slots:
|
||||
await _set_step(guide_id, 2, "Checking content…")
|
||||
sections_per_chunk = {
|
||||
i: "\n\n".join(f"SECTION: {_title(entries[num])}\n{content_by_num[num]}" for num in content_chunk_nums[i])
|
||||
for i, _ in pending_slots
|
||||
}
|
||||
slots = [{
|
||||
"key": f"{guide_id}-content-check-{i + 1}-j{j + 1}",
|
||||
"prompt": _prompt(
|
||||
"Guide-Content-Check",
|
||||
topic=topic, format_name=format_name, sections=sections_per_chunk[i],
|
||||
out_path=check_judge_paths[i][j], extra=_extra(instructions),
|
||||
),
|
||||
"role": "judge", "capabilities": "files",
|
||||
"payload": (lambda result, p=check_judge_paths[i][j]: _read_problems_schema(_json_file(p))),
|
||||
} for i, j in pending_slots]
|
||||
n_checks = len(slots)
|
||||
upd = lambda n: asyncio.create_task(_set_step(guide_id, 2, f"Checking content {n}/{n_checks}…"))
|
||||
await _race(topic, "Content-Exam", slots, len(slots), _timeout("content_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||||
if is_cancelled():
|
||||
return None
|
||||
|
||||
problems_by_num: dict[int, str] = {}
|
||||
for i in range(writer_count):
|
||||
if content_chunk_nums[i]:
|
||||
problems_by_num.update(_panel_problems(check_judge_paths[i], set(content_chunk_nums[i]), idx))
|
||||
|
||||
if problems_by_num:
|
||||
_log(topic, f"Content exam: {len(problems_by_num)} block(s) flagged")
|
||||
await _set_step(guide_id, 2, f"Revising {len(problems_by_num)} content(s)…")
|
||||
fix_chunks = [[num for num in nums if num in problems_by_num] for nums in content_chunk_nums]
|
||||
fix_paths = [content_path.parent / f"{content_path.stem}.content-fix-{i + 1}.md" for i in range(writer_count)]
|
||||
fix_pending = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()]
|
||||
results = await asyncio.gather(*[
|
||||
run_agent(
|
||||
f"{guide_id}-content-fix-{i + 1}",
|
||||
_prompt(
|
||||
"Guide-Content-Fix",
|
||||
topic=topic, facts=facts,
|
||||
tasks="\n\n".join(
|
||||
f"SECTION: {_title(entries[num])}\nPROBLEM: {problems_by_num[num]}\nCURRENT:\n{content_by_num[num]}"
|
||||
for num in fix_chunks[i]
|
||||
),
|
||||
out_path=fix_paths[i], extra=_extra(instructions),
|
||||
),
|
||||
_timeout("content", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full",
|
||||
)
|
||||
for i in fix_pending
|
||||
], return_exceptions=True)
|
||||
if is_cancelled():
|
||||
return None
|
||||
for p in fix_paths:
|
||||
if not p.exists():
|
||||
continue
|
||||
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
|
||||
num = _resolve_title(idx, sec["title"])
|
||||
if num in problems_by_num and sec["md"].strip():
|
||||
content_by_num[num] = sec["md"]
|
||||
|
||||
_set_done(content_path, 2) # content check done
|
||||
|
||||
# Step 4: writing — the writer phrases out the checked content (resume).
|
||||
# FINE chunks: exactly 1 block per writer → variable lengths, no budget rationing.
|
||||
def content_text(chunk) -> str:
|
||||
nums = [num for ch in chunk for num in ch["nums"] if num in content_by_num]
|
||||
return "\n\n".join(f"<!-- section: {_title(entries[num])} -->\n{content_by_num[num]}" for num in nums)
|
||||
|
||||
w_chunks = [[{"title": ch["title"], "nums": [num]}] for ch in plan for num in ch["nums"]]
|
||||
w_assignments = [_assignment_subs(c, entries, subs_by_title) for c in w_chunks]
|
||||
paths = [content_path.parent / f"{content_path.stem}.chunk-{i}.md" for i in range(1, len(w_chunks) + 1)]
|
||||
pending = [i for i, p in enumerate(paths) if not p.exists()]
|
||||
if pending:
|
||||
async def report(d, t): await _set_step(guide_id, 3, f"Writing sections {d}/{t}…")
|
||||
results = await _gather_progress([
|
||||
run_agent(
|
||||
f"{guide_id}-w{i + 1}",
|
||||
_prompt(
|
||||
"Guide-Writer",
|
||||
topic=topic, format_name=format_name, assignment=w_assignments[i],
|
||||
contents=content_text(w_chunks[i]),
|
||||
spec=spec, out_path=paths[i], extra=_extra(instructions),
|
||||
),
|
||||
_timeout("writer", 1), provider=provider, role="guide", capabilities="files",
|
||||
)
|
||||
for i in pending
|
||||
], len(w_chunks), report, start=len(w_chunks) - len(pending))
|
||||
if is_cancelled():
|
||||
return None
|
||||
for i, r in zip(pending, results):
|
||||
if isinstance(r, BaseException):
|
||||
_log(topic, f"Writer {i + 1}: {type(r).__name__}: {r}")
|
||||
elif r[0] != 0:
|
||||
_log(topic, f"Writer {i + 1}: {_claude_error('Error', *r)}")
|
||||
elif not paths[i].exists():
|
||||
_log(topic, f"Writer {i + 1}: no output file created")
|
||||
if not any(p.exists() for p in paths):
|
||||
await _fail(guide_id, _gather_error("Writer error", list(results)))
|
||||
return None
|
||||
|
||||
by_num: dict[int, dict] = {}
|
||||
for p in paths:
|
||||
if not p.exists():
|
||||
continue
|
||||
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
|
||||
num = _resolve_title(idx, sec["title"])
|
||||
if num is None:
|
||||
_log(topic, f"Writer produced unknown section '{sec['title'][:40]}' (ignored)")
|
||||
elif num not in by_num:
|
||||
by_num[num] = sec
|
||||
if not by_num:
|
||||
await _fail(guide_id, "No sections found in writer output")
|
||||
return None
|
||||
|
||||
# Follow-up round: write missing sections (writer failure) deliberately — one round.
|
||||
missing_after = [num for num in planned_nums if num not in by_num]
|
||||
if missing_after:
|
||||
_log(topic, f"Writing: {len(missing_after)} section(s) missing — follow-up round…")
|
||||
nw_chunks = [[{"title": "Additional", "nums": [num]}] for num in missing_after]
|
||||
nw_paths = [content_path.parent / f"{content_path.stem}.chunk-nach-{k}.md" for k in range(1, len(nw_chunks) + 1)]
|
||||
nw_pending = [k for k, p in enumerate(nw_paths) if not p.exists()]
|
||||
if nw_pending:
|
||||
async def report_nw(d, t): await _set_step(guide_id, 3, f"Writing missing sections {d}/{t}…")
|
||||
await _gather_progress([
|
||||
run_agent(
|
||||
f"{guide_id}-w-nach-{k + 1}",
|
||||
_prompt(
|
||||
"Guide-Writer",
|
||||
topic=topic, format_name=format_name, assignment=_assignment_subs(nw_chunks[k], entries, subs_by_title),
|
||||
contents=content_text(nw_chunks[k]), spec=spec, out_path=nw_paths[k], extra=_extra(instructions),
|
||||
),
|
||||
_timeout("writer", 1), provider=provider, role="guide", capabilities="files",
|
||||
)
|
||||
for k in nw_pending
|
||||
], len(nw_chunks), report_nw, start=len(nw_chunks) - len(nw_pending))
|
||||
if is_cancelled():
|
||||
return None
|
||||
for p in nw_paths:
|
||||
if not p.exists():
|
||||
continue
|
||||
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
|
||||
num = _resolve_title(idx, sec["title"])
|
||||
if num is not None and num not in by_num and sec["md"].strip():
|
||||
by_num[num] = sec
|
||||
|
||||
if all(p.exists() for p in paths):
|
||||
_set_done(content_path, 3) # writing complete
|
||||
|
||||
# Step 3: reading-exam loop — check per writer packet, fix only for
|
||||
# flagged sections; follow-up rounds check ONLY the replaced sections.
|
||||
# After the round cap, open complaints stand.
|
||||
chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in by_num] for chunk in chunks]
|
||||
|
||||
def sections_text(nums: list[int]) -> str:
|
||||
return "\n\n".join(f"SECTION: {_title(entries[num])}\n{by_num[num]['md']}" for num in nums)
|
||||
|
||||
def _sub_list(num: int) -> str:
|
||||
subs = subs_by_title.get(_title(entries[num]), [])
|
||||
return "\n".join(f"- [{_level_label(s)}] {s['title']}" for s in subs) or "(none)"
|
||||
|
||||
def tasks_text(nums: list[int], problems: dict[int, str]) -> str:
|
||||
return "\n\n".join(
|
||||
f"SECTION: {_title(entries[num])}\n"
|
||||
f"SUBBLOCKS (set one `<!-- sub: LABEL | title -->` marker each, label/order as here):\n{_sub_list(num)}\n"
|
||||
f"PROBLEM: {problems[num]}\nCURRENT CONTENT:\n{by_num[num]['md']}"
|
||||
for num in nums
|
||||
)
|
||||
|
||||
scope = chunk_nums
|
||||
for round_no in range(1, READING_ROUNDS + 1):
|
||||
# CHECK_PANEL judges per packet; majority flags. Aggregation robust against a single failure.
|
||||
check_judge_paths = [
|
||||
[content_path.parent / f"{content_path.stem}.lese-check-r{round_no}-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)]
|
||||
for i in range(1, writer_count + 1)
|
||||
]
|
||||
pending_slots = [
|
||||
(i, j) for i in range(writer_count) if scope[i]
|
||||
for j in range(CHECK_PANEL) if _read_problems_schema(_json_file(check_judge_paths[i][j])) is None
|
||||
]
|
||||
if pending_slots:
|
||||
await _set_step(guide_id, 4, "Checking readability…")
|
||||
sections_per_chunk = {i: sections_text(scope[i]) for i, _ in pending_slots}
|
||||
slots = [{
|
||||
"key": f"{guide_id}-lese-check-r{round_no}-{i + 1}-j{j + 1}",
|
||||
"prompt": _prompt(
|
||||
"Guide-Lese-Check",
|
||||
topic=topic, format_name=format_name, spec=spec,
|
||||
sections=sections_per_chunk[i],
|
||||
out_path=check_judge_paths[i][j], extra=_extra(instructions),
|
||||
),
|
||||
"role": "judge", "capabilities": "files",
|
||||
"payload": (lambda result, p=check_judge_paths[i][j]: _read_problems_schema(_json_file(p))),
|
||||
} for i, j in pending_slots]
|
||||
n_checks = len(slots)
|
||||
upd = lambda n: asyncio.create_task(_set_step(guide_id, 4, f"Checking readability {n}/{n_checks}…"))
|
||||
res = await _race(topic, f"Reading-Exam r{round_no}", slots, len(slots), _timeout("lese_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||||
if is_cancelled():
|
||||
return None
|
||||
if res is None:
|
||||
_log(topic, f"Reading exam round {round_no}: no full quorum — aggregated available judges")
|
||||
|
||||
problems_by_num: dict[int, str] = {}
|
||||
for i in range(writer_count):
|
||||
if scope[i]:
|
||||
problems_by_num.update(_panel_problems(check_judge_paths[i], set(scope[i]), idx))
|
||||
|
||||
# Deterministic readability gate: queue too-hard sections into the same
|
||||
# revision (LLM complaint takes precedence). Gate off → no-op.
|
||||
if READABILITY_ACTIVE:
|
||||
md_by_num = {num: by_num[num]["md"] for nums in scope for num in nums if num in by_num}
|
||||
hints = await asyncio.to_thread(readability.rate_sections, md_by_num)
|
||||
if hints:
|
||||
_log(topic, f"Readability: {len(hints)} section(s) too hard")
|
||||
for num, hint in hints.items():
|
||||
problems_by_num.setdefault(num, hint)
|
||||
|
||||
if not problems_by_num:
|
||||
break
|
||||
|
||||
_log(topic, f"Reading exam round {round_no}: {len(problems_by_num)} section(s) flagged")
|
||||
await _set_step(guide_id, 4, f"Revising {len(problems_by_num)} section(s) (round {round_no})…")
|
||||
fix_chunks = [[num for num in nums if num in problems_by_num] for nums in chunk_nums]
|
||||
fix_paths = [content_path.parent / f"{content_path.stem}.fix-r{round_no}-{i + 1}.md" for i in range(writer_count)]
|
||||
fix_pending = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()]
|
||||
results = await asyncio.gather(*[
|
||||
run_agent(
|
||||
f"{guide_id}-fix-r{round_no}-w{i + 1}",
|
||||
_prompt(
|
||||
"Guide-Sections-Fix",
|
||||
topic=topic, format_name=format_name, facts=facts, spec=spec,
|
||||
tasks=tasks_text(fix_chunks[i], problems_by_num),
|
||||
out_path=fix_paths[i], extra=_extra(instructions),
|
||||
),
|
||||
_timeout("writer", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full",
|
||||
)
|
||||
for i in fix_pending
|
||||
], return_exceptions=True)
|
||||
if is_cancelled():
|
||||
return None
|
||||
for i, r in zip(fix_pending, results):
|
||||
if isinstance(r, BaseException) or (not isinstance(r, BaseException) and r[0] != 0):
|
||||
_log(topic, f"Sections fix {i + 1} (round {round_no}) failed — original kept")
|
||||
replaced: set[int] = set()
|
||||
for p in fix_paths:
|
||||
if not p.exists():
|
||||
continue
|
||||
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
|
||||
num = _resolve_title(idx, sec["title"])
|
||||
if num not in problems_by_num or not sec["md"].strip():
|
||||
continue
|
||||
# Marker invariant: if the fix loses the sub markers although the original had
|
||||
# some, it is discarded — otherwise the level filter (E/M/S/F) dies silently.
|
||||
if by_num[num].get("subs") and not sec.get("subs"):
|
||||
_log(topic, f"Reading fix for '{sec['title']}' without sub markers — discarded, tagged original kept")
|
||||
continue
|
||||
by_num[num] = sec
|
||||
replaced.add(num)
|
||||
_log(topic, f"Reading exam round {round_no}: {len(replaced)} section(s) revised")
|
||||
if not replaced:
|
||||
break
|
||||
if round_no == READING_ROUNDS:
|
||||
_log(topic, f"Reading exam: 1 round — revision stays unchecked")
|
||||
break
|
||||
scope = [[num for num in nums if num in replaced] for nums in chunk_nums]
|
||||
_set_done(content_path, 4) # reading exam done
|
||||
|
||||
# Checkable = format has an exam AND the block has ≥1 relevant subblock.
|
||||
# Guide is always checkable (even without relevance data, fallback = everything).
|
||||
def _checkable(num):
|
||||
if format_name == "Guide":
|
||||
return True
|
||||
if format_name == "FullGuide":
|
||||
return any(isinstance(s, dict) and s.get("relevance") == "relevant"
|
||||
for s in subs_raw.get(_title(entries[num]), []))
|
||||
return False # Rest etc. → pure reading sections
|
||||
|
||||
await _set_progress(guide_id, "Assembling…")
|
||||
chapters: list[dict] = []
|
||||
for ch in plan:
|
||||
sections = [
|
||||
{"num": num, "title": _title(entries[num]), "md": by_num[num]["md"],
|
||||
"compact": by_num[num].get("compact", ""),
|
||||
"anchor": by_num[num].get("anchor", ""), "anker_compact": by_num[num].get("anker_compact", ""),
|
||||
"subs": by_num[num].get("subs", []), "checkable": _checkable(num)}
|
||||
for num in ch["nums"] if num in by_num
|
||||
]
|
||||
if sections:
|
||||
chapters.append({"title": ch["title"], "sections": sections})
|
||||
planned = {num for ch in plan for num in ch["nums"]}
|
||||
missing = sorted(planned - set(by_num))
|
||||
if missing:
|
||||
_log(topic, f"Sections missing from writer output: {[_title(entries[n]) for n in missing]}")
|
||||
if not chapters:
|
||||
await _fail(guide_id, "No sections found in writer output")
|
||||
return None
|
||||
return chapters
|
||||
|
||||
|
||||
_LEVEL_RANK = {"beginner": 1, "advanced": 2, "expert": 3, "peripheral": 4,
|
||||
@@ -892,14 +253,18 @@ async def generate_guide(guide_id: str, topic: str, format_name: str, instructio
|
||||
if project:
|
||||
await asyncio.to_thread(_convert_pdfs, project)
|
||||
|
||||
# Re-run from step: delete content + slots from `ab_step`, rest stays → resume rebuilds from there.
|
||||
# Otherwise "recreate": a finished guide → complete fresh start.
|
||||
# Otherwise step files are leftovers of an abort/error → resume.
|
||||
import guide_board # lazy — guide_board imports helpers from this module
|
||||
# Re-run from stage: cards from `ab_step` onward back to that column.
|
||||
# A FINISHED guide without ab_step → complete fresh start (board + slots wiped).
|
||||
# Otherwise cards are leftovers of an abort/error → resume at their stored stage.
|
||||
if ab_step is not None:
|
||||
_reset_guide_from_step(content_path, ab_step)
|
||||
await guide_board.reset_from_stage(topic, format_name, ab_step)
|
||||
elif content_path.exists():
|
||||
for p_alt in guide_slot_files(content_path):
|
||||
p_alt.unlink(missing_ok=True)
|
||||
counts = await guide_stage_counts(topic, format_name)
|
||||
if not counts or set(counts) == {"done"}:
|
||||
await delete_guide_board(topic, format_name)
|
||||
for p_alt in guide_slot_files(content_path):
|
||||
p_alt.unlink(missing_ok=True)
|
||||
|
||||
bs = await list_blocks(topic, status="consensus")
|
||||
if bs:
|
||||
@@ -912,12 +277,13 @@ async def generate_guide(guide_id: str, topic: str, format_name: str, instructio
|
||||
await _fail(guide_id, "No blocks found")
|
||||
return
|
||||
entries = _unique_title(alle)
|
||||
facts = _prompt("Guide-Facts-Projekt", project=project) if project else _prompt("Guide-Facts-Thema")
|
||||
chapters = await _generate_sections(
|
||||
guide_id, topic, format_name, entries,
|
||||
facts, instructions, provider, content_path,
|
||||
chapters = await guide_board.run_guide_board(
|
||||
guide_id, topic, format_name, entries, instructions, provider, content_path,
|
||||
)
|
||||
if chapters is None or is_guide_cancelled(guide_id):
|
||||
if is_guide_cancelled(guide_id):
|
||||
return
|
||||
if chapters is None:
|
||||
await _fail(guide_id, "No finished sections (see board — cards with errors)")
|
||||
return
|
||||
content = {"topic": topic, "format": format_name, "chapters": chapters}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user