import argparse import difflib import posixpath import re import subprocess import sys from pathlib import Path from check_conventions import DEBT as INDEX, debt_name DEBT_DIR = "docs/dev/debt" HEADING = re.compile(r"^(#{1,6}) (.+?)\s*$") LINK = re.compile(r"\]\(([^)\s]+)\)") def parse(text): items = [] section = None current = None for line in text.splitlines(): heading = HEADING.match(line) if heading: section = heading[2] if len(heading[1]) == 3 else section current = None items.append(["heading", section, [line]]) elif line.startswith("- "): current = ["entry", section, [line]] items.append(current) elif current and (line.startswith(" ") or line.startswith("\t")): current[2].append(line) elif line.strip(): current = None items.append(["text", section, [line]]) else: current = None return [(kind, section, "\n".join(lines)) for kind, section, lines in items] def key(kind, text): return debt_name(text) if kind == "entry" else None def relink(text, source, destination): def replace(match): target = match.group(1) if re.match(r"^[a-z][a-z0-9+.-]*:", target): return match.group(0) path, _, anchor = target.partition("#") resolved = posixpath.normpath(posixpath.join(posixpath.dirname(source), path)) if path else source moved = posixpath.relpath(resolved, posixpath.dirname(destination)) return "](" + moved + ("#" + anchor if anchor else "") + ")" return LINK.sub(replace, text) def debt_file(name): return f"{DEBT_DIR}/{name}.md" def render(title, sections, order): out = [title, ""] for section in sorted(sections, key=lambda section: order.index(section) if section in order else len(order)): out += [f"### {section}", ""] + sections[section] + [""] return "\n".join(out) def read_debt(root, name): path = root / debt_file(name) sections = {} if path.exists(): section = None for kind, item_section, text in parse(path.read_text(encoding="utf-8")): if kind == "heading" and item_section and text.startswith("### "): section = item_section sections.setdefault(section, []) elif kind != "heading" and section: sections[section].append(text) return sections def write_debt(root, name, sections, order): path = root / debt_file(name) sections = {section: entries for section, entries in sections.items() if entries} if not sections: if path.exists(): path.unlink() return path.parent.mkdir(parents=True, exist_ok=True) path.write_text(render(f"# {name} technical debt", sections, order), encoding="utf-8") def sections_of(text): return [section for kind, section, _ in parse(text) if kind == "heading" and section] def split(text): order = sections_of(text) index, files = [], {} for kind, section, item in parse(text): name = key(kind, item) if name is None: index.append((kind, item)) else: files.setdefault(name, {}).setdefault(section, []).append(relink(item, INDEX, debt_file(name))) return index, files, order def render_index(index): out = [] for kind, item in index: if kind == "heading" and out and out[-1] != "": out.append("") out.append(item) if kind == "heading" or kind == "text": out.append("") return "\n".join(out).rstrip("\n") + "\n" def index_note(): return ("text", f"Entries about one library or component are in [debt](debt/), one file per library " "or component, under the same sections as this page. Entries that belong to none stay here.") def command_split(root): text = (root / INDEX).read_text(encoding="utf-8") index, files, order = split(text) if not files: sys.exit(f"{INDEX} has no entries to move") for name, sections in files.items(): existing = read_debt(root, name) for section, entries in sections.items(): existing.setdefault(section, []).extend(entries) write_debt(root, name, existing, order) if index_note() not in index: index.insert(1, index_note()) (root / INDEX).write_text(render_index(index), encoding="utf-8") print("\n".join(sorted(debt_file(name) for name in files))) def git(root, *args, check=True): return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True, encoding="utf-8", check=check).stdout def apply_change(root, base, new, index_text): order = sections_of(index_text) or sections_of(new) before = [(kind, section, item, key(kind, item)) for kind, section, item in parse(base) if kind == "entry" and key(kind, item)] after = [(kind, section, item, key(kind, item)) for kind, section, item in parse(new) if kind == "entry" and key(kind, item)] files = {} def load(name): if name not in files: files[name] = read_debt(root, name) return files[name] def locate(entry): _, section, item, name = entry entries = load(name).get(section, []) moved = relink(item, INDEX, debt_file(name)) if moved not in entries: raise LookupError(f"{debt_file(name)}: entry not found under '{section}': {item.splitlines()[0][:100]}") return entries, entries.index(moved) def insert(entry, anchor): _, section, item, name = entry entries = load(name).setdefault(section, []) moved = relink(item, INDEX, debt_file(name)) position = 0 if anchor is None else len(entries) if anchor is not None and anchor[1] == section and anchor[3] == name: anchored = relink(anchor[2], INDEX, debt_file(name)) if anchored in entries: position = entries.index(anchored) + 1 entries.insert(position, moved) matcher = difflib.SequenceMatcher(None, [entry[1:3] for entry in before], [entry[1:3] for entry in after], autojunk=False) for tag, i1, i2, j1, j2 in matcher.get_opcodes(): if tag == "equal": continue removed, added = before[i1:i2], after[j1:j2] kept_old, kept_new = set(), set() for offset, (old, fresh) in enumerate(zip(removed, added)): if old[1] == fresh[1] and old[3] == fresh[3]: entries, position = locate(old) entries[position] = relink(fresh[2], INDEX, debt_file(fresh[3])) kept_old.add(offset) kept_new.add(offset) for offset, old in enumerate(removed): if offset not in kept_old: entries, position = locate(old) del entries[position] for offset, fresh in enumerate(added): if offset in kept_new: continue anchor = after[j1 + offset - 1] if j1 + offset > 0 else None insert(fresh, anchor) for name, sections in files.items(): write_debt(root, name, sections, order) return sorted(debt_file(name) for name in files) def unkeyed(text): return "\n".join(item for kind, _, item in parse(text) if key(kind, item) is None) def command_resolve(root): stages = {} for line in git(root, "ls-files", "-u", "--", INDEX).splitlines(): meta, _ = line.split("\t", 1) _, sha, stage = meta.split() stages[stage] = sha if not {"1", "2", "3"} <= set(stages): sys.exit(f"{INDEX} is not in a three-way conflict") base, ours, theirs = (git(root, "cat-file", "-p", stages[stage]) for stage in "123") if unkeyed(base) != unkeyed(theirs): sys.exit(f"{INDEX}: the change also edits lines that stay in {INDEX}; resolve those by hand") try: touched = apply_change(root, base, theirs, ours) except LookupError as error: sys.exit(str(error)) (root / INDEX).write_text(ours, encoding="utf-8") git(root, "add", "--", INDEX, *touched) print("\n".join(touched)) def main(): parser = argparse.ArgumentParser(description=f"Split {INDEX} into one file per library or component, and move entries of a change made before the split") parser.add_argument("command", choices=("split", "resolve"), help=f"split: move the entries of {INDEX} into {DEBT_DIR}/; resolve: during a rebase stopped on {INDEX}, apply the stopped commit's entry changes to {DEBT_DIR}/ and stage them") args = parser.parse_args() root = Path(git(Path.cwd(), "rev-parse", "--show-toplevel").strip()) command_split(root) if args.command == "split" else command_resolve(root) if __name__ == "__main__": main()