Module paroxython.recommend_programs

A wrapper around ProgramFilter, feeding it with commands and assessing the learning costs of its final state.

Description

The recommendation system is initialized with a database created by paroxython.make_db. It parses the given pipeline (example), checks its commands and submits them to the filter, which in response updates these sets:

• selected_programs, the names of the recommended programs;
• imparted_knowledge, the taxa modelizing the imparted knowledge;
• hidden_programs/hidden_taxa, the programs/taxa to be removed from the final output.

When the pipeline is exhausted, it retrieves their values, compute the associated learning costs, and produces a Markdown report (example) whose each section consists in:

• the name of a recommended program;
• its total learning cost;
• its source code;
• the table of its taxa, lexicographically sorted, and accompanied by their spans and individual learning cost.

To maximize their utility, these sections are grouped by cost range, and each group is sorted by increasing total cost and SLOC.

import subprocess
from collections import defaultdict
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union

import regex  # type: ignore

from .assess_costs import LearningCostAssessor
from .filter_programs import ProgramFilter
from .goodies import (
    add_line_numbers,
    cost_bucket,
    couple_to_string,
    enumeration_to_txt_factory,
    print_warning,
    title_to_slug_factory,
)
from .user_types import (
    AssessedPrograms,
    Command,
    Criterion,
    JsonTagDatabase,
    Operation,
    ProgramPaths,
)

__pdoc__ = {
    "MalformedData": False,
    "Recommendations": "",
    "Recommendations.__init__": True,
}


class MalformedData(Exception):
    ...


class Recommendations:
    def __init__(
        self,
        db: JsonTagDatabase,
        base_path: Optional[Path] = None,
        title_format: str = "`{name}`",
        **kwargs,
    ) -> None:
        self.base_path = base_path
        self.title_format = title_format

        # copy locally some attributes and methods of a ProgramFilter instance
        program_filter = ProgramFilter(db)
        self.db_programs = program_filter.db_programs
        self.selected_programs = program_filter.selected_programs
        self.imparted_knowledge = program_filter.imparted_knowledge
        self.hidden_taxa = program_filter.hidden_taxa
        self.hidden_programs = program_filter.hidden_programs
        self.update_filter = program_filter.update_filter

        self.assess = LearningCostAssessor(self.db_programs, **kwargs)
        self.result: List[Tuple[int, Operation, ProgramPaths]] = []

    def run_pipeline(self, commands: Optional[List[Command]] = None) -> None:
        commands = commands or []
        if not isinstance(commands, list):
            print_warning(f"commands are ignored (should be a list).")
            commands = []
        current = set(self.db_programs)
        print(f"\nProcessing {len(commands)} commands on {len(current)} programs.")

        # Execute sequentially all the commands of the pipeline
        for (i, command) in enumerate(commands, 1):

            # Retrieve the operation
            try:
                operation = command["operation"]
            except KeyError:
                print_warning(f"operation {i} is ignored (not specified).")
                continue
            except TypeError:
                print_warning(f"operation {i} is ignored (malformed).")
                continue
            (operation, n) = regex.subn(" all", "", operation)
            quantifier = "all" if n == 1 else "any"
            operation = Operation(operation.replace(" any", ""))
            if operation not in ("include", "exclude", "impart", "hide"):
                print_warning(f"operation {i} ({operation}) is ignored (unknown).")
                continue

            # Retrieve the data
            try:
                data = self.parse_data(command.get("data", []), operation)
            except MalformedData:
                print_warning(f"unable to interpret the data of operation {i} ({operation}).")
                continue
            if not data:
                print_warning(f"operation {i} ({operation}) is ignored (no data).")
                continue

            # Update the state of the filter: selected / hidden programs, imparted / hidden taxa
            self.update_filter(data, operation, quantifier)

            # Log the statistics of the filter state for the last operation
            (previous, current) = (current, set(self.selected_programs))
            self.result.append((i, operation, sorted(previous - current)))
            print(f"  {len(current)} programs remaining after operation {i} ({operation}).")

        self.assess.set_imparted_knowledge(self.imparted_knowledge)
        self.assessed_programs = self.assess(self.selected_programs)

    def parse_data(
        self,
        data: Union[str, List[Criterion]],
        operation: Operation,
    ) -> List[Criterion]:
        if isinstance(data, list):  # The object can either be a list of criteria...
            return data
        elif isinstance(data, str):  # ... or a shell command printing them on stdout
            criteria = subprocess.run(
                str(data).format(base_path=self.base_path),  # str() needed by mypy
                stdout=subprocess.PIPE,  # From Python 3.7, these two arguments can be
                stderr=subprocess.PIPE,  # ... replaced by: capture_output=True
                encoding="utf-8",
                shell=True,
                check=True,
            )
            return [criterion for criterion in criteria.stdout.strip().split("\n")]
        else:
            raise MalformedData

    def get_markdown(
        self,
        span_column_width: int = 30,
        sorting_strategy: str = "by_cost_and_sloc",
        grouping_strategy: str = "by_cost_bucket",
    ) -> str:
        # Define some helper functions.

        title_to_slug = title_to_slug_factory()
        spans_to_html = enumeration_to_txt_factory(span_column_width, "_imported_")
        display_count = lambda n: f"{n} program" + ("" if n == 1 else "s")

        sorting_key = None
        if sorting_strategy == "by_cost_and_sloc":
            sorting_key = lambda x: (x[0], len(self.db_programs[x[1]]["source"].split("\n")))
        elif sorting_strategy == "lexicographic":
            sorting_key = lambda x: x[1]

        grouping_key = lambda x: "(default: no group)"
        if grouping_strategy == "by_cost_bucket":
            grouping_key = cost_bucket

        # Group resulting programs into cost buckets, and sort each group by increasing difficulty.

        toc_data: Dict[str, AssessedPrograms] = defaultdict(list)
        for (cost, program_path) in self.assessed_programs:
            if program_path in self.hidden_programs:
                continue
            toc_data[grouping_key(cost)].append((cost, program_path))
        for costs_and_program_paths in toc_data.values():
            costs_and_program_paths.sort(key=sorting_key)

        # Accumulate simultaneously the TOC and the contents.

        toc: List[str] = ["# Table of contents"]
        contents: List[str] = ["# Recommended programs"]

        for (bounds, costs_and_program_paths) in toc_data.items():

            title = f"{display_count(len(costs_and_program_paths))} of learning cost {bounds}"
            toc.append(f"- [`{title}`](#{title_to_slug(title)})")
            contents.append(f"\n## {title}")

            for (cost, program_path) in costs_and_program_paths:
                program_info = self.db_programs[program_path]
                name = regex.sub(r"(?:.+/)?(.+)", r"\1", program_path)  # last segment of the path
                program_title = self.title_format.format(path=program_path, name=name)
                title = f"Program {program_title} (learning cost {cost})"
                slug = title_to_slug(f"Program {name} (learning cost {cost})")
                toc.append(f"    - [`{name}`](#{slug})")
                contents.append(f"\n### {title}")
                contents.append(f"\n```python\n{add_line_numbers(program_info['source'])}\n```")
                contents.append("\n| Cost  | Taxon | Location |")
                contents.append("|" + "----|" * 3)
                items = sorted(
                    program_info["taxa"].items(),
                    key=lambda x: f"~{x[0]}" if x[0].startswith("meta/") else x[0],
                )
                for (taxon_name, spans) in items:
                    if taxon_name in self.hidden_taxa:
                        continue
                    taxon_cost = self.assess.taxon_cost(taxon_name)
                    s = spans_to_html(", ".join(map(couple_to_string, spans)))
                    contents.append(f"| {taxon_cost} | `{taxon_name}` | {s} |")
                contents.append("\n---")

        def programs_to_html(description: str, programs: ProgramPaths) -> str:
            details = []
            if programs:
                description = description.replace(" 1 programs", " 1 program")
                details = ["<ol>"]
                for program in sorted(programs):
                    details.append(f"  <li><code>{program}</code></li>")
                details.append("</ol>")
            return (
                "<details>\n"
                + f"  <summary>{description}.</summary>\n  "
                + "\n  ".join(details)
                + "\n</details>\n"
            )

        n = len(self.db_programs)
        summary: List[str] = ["\n# Summary"]
        summary.append(programs_to_html(f"{n} initially", list(self.db_programs)))
        for (i, operation, removed_programs) in self.result:
            n -= len(removed_programs)
            title = f"{n} remaining after operation {i} ({operation}) has filtered out {len(removed_programs)} programs"
            summary.append(programs_to_html(title, removed_programs))

        return "\n".join(toc + contents + summary)


if __name__ == "__main__":
    ast = __import__("ast")
    json = __import__("json")
    path = Path("../algo/programs")
    rec = Recommendations(
        db=json.loads(Path(f"{path}_db.json").read_text()),
        base_path=path.parent,
    )
    rec.run_pipeline(ast.literal_eval(Path(f"{path}_pipe.py").read_text()))
    output_path = Path(f"{path}_recommendations.md")
    output_path.write_text(rec.get_markdown())
    print(f"Dumped: {output_path.resolve()}\n")

Classes

class Recommendations

Expand source code Browse GitHub

class Recommendations:
    def __init__(
        self,
        db: JsonTagDatabase,
        base_path: Optional[Path] = None,
        title_format: str = "`{name}`",
        **kwargs,
    ) -> None:
        self.base_path = base_path
        self.title_format = title_format

        # copy locally some attributes and methods of a ProgramFilter instance
        program_filter = ProgramFilter(db)
        self.db_programs = program_filter.db_programs
        self.selected_programs = program_filter.selected_programs
        self.imparted_knowledge = program_filter.imparted_knowledge
        self.hidden_taxa = program_filter.hidden_taxa
        self.hidden_programs = program_filter.hidden_programs
        self.update_filter = program_filter.update_filter

        self.assess = LearningCostAssessor(self.db_programs, **kwargs)
        self.result: List[Tuple[int, Operation, ProgramPaths]] = []

    def run_pipeline(self, commands: Optional[List[Command]] = None) -> None:
        commands = commands or []
        if not isinstance(commands, list):
            print_warning(f"commands are ignored (should be a list).")
            commands = []
        current = set(self.db_programs)
        print(f"\nProcessing {len(commands)} commands on {len(current)} programs.")

        # Execute sequentially all the commands of the pipeline
        for (i, command) in enumerate(commands, 1):

            # Retrieve the operation
            try:
                operation = command["operation"]
            except KeyError:
                print_warning(f"operation {i} is ignored (not specified).")
                continue
            except TypeError:
                print_warning(f"operation {i} is ignored (malformed).")
                continue
            (operation, n) = regex.subn(" all", "", operation)
            quantifier = "all" if n == 1 else "any"
            operation = Operation(operation.replace(" any", ""))
            if operation not in ("include", "exclude", "impart", "hide"):
                print_warning(f"operation {i} ({operation}) is ignored (unknown).")
                continue

            # Retrieve the data
            try:
                data = self.parse_data(command.get("data", []), operation)
            except MalformedData:
                print_warning(f"unable to interpret the data of operation {i} ({operation}).")
                continue
            if not data:
                print_warning(f"operation {i} ({operation}) is ignored (no data).")
                continue

            # Update the state of the filter: selected / hidden programs, imparted / hidden taxa
            self.update_filter(data, operation, quantifier)

            # Log the statistics of the filter state for the last operation
            (previous, current) = (current, set(self.selected_programs))
            self.result.append((i, operation, sorted(previous - current)))
            print(f"  {len(current)} programs remaining after operation {i} ({operation}).")

        self.assess.set_imparted_knowledge(self.imparted_knowledge)
        self.assessed_programs = self.assess(self.selected_programs)

    def parse_data(
        self,
        data: Union[str, List[Criterion]],
        operation: Operation,
    ) -> List[Criterion]:
        if isinstance(data, list):  # The object can either be a list of criteria...
            return data
        elif isinstance(data, str):  # ... or a shell command printing them on stdout
            criteria = subprocess.run(
                str(data).format(base_path=self.base_path),  # str() needed by mypy
                stdout=subprocess.PIPE,  # From Python 3.7, these two arguments can be
                stderr=subprocess.PIPE,  # ... replaced by: capture_output=True
                encoding="utf-8",
                shell=True,
                check=True,
            )
            return [criterion for criterion in criteria.stdout.strip().split("\n")]
        else:
            raise MalformedData

    def get_markdown(
        self,
        span_column_width: int = 30,
        sorting_strategy: str = "by_cost_and_sloc",
        grouping_strategy: str = "by_cost_bucket",
    ) -> str:
        # Define some helper functions.

        title_to_slug = title_to_slug_factory()
        spans_to_html = enumeration_to_txt_factory(span_column_width, "_imported_")
        display_count = lambda n: f"{n} program" + ("" if n == 1 else "s")

        sorting_key = None
        if sorting_strategy == "by_cost_and_sloc":
            sorting_key = lambda x: (x[0], len(self.db_programs[x[1]]["source"].split("\n")))
        elif sorting_strategy == "lexicographic":
            sorting_key = lambda x: x[1]

        grouping_key = lambda x: "(default: no group)"
        if grouping_strategy == "by_cost_bucket":
            grouping_key = cost_bucket

        # Group resulting programs into cost buckets, and sort each group by increasing difficulty.

        toc_data: Dict[str, AssessedPrograms] = defaultdict(list)
        for (cost, program_path) in self.assessed_programs:
            if program_path in self.hidden_programs:
                continue
            toc_data[grouping_key(cost)].append((cost, program_path))
        for costs_and_program_paths in toc_data.values():
            costs_and_program_paths.sort(key=sorting_key)

        # Accumulate simultaneously the TOC and the contents.

        toc: List[str] = ["# Table of contents"]
        contents: List[str] = ["# Recommended programs"]

        for (bounds, costs_and_program_paths) in toc_data.items():

            title = f"{display_count(len(costs_and_program_paths))} of learning cost {bounds}"
            toc.append(f"- [`{title}`](#{title_to_slug(title)})")
            contents.append(f"\n## {title}")

            for (cost, program_path) in costs_and_program_paths:
                program_info = self.db_programs[program_path]
                name = regex.sub(r"(?:.+/)?(.+)", r"\1", program_path)  # last segment of the path
                program_title = self.title_format.format(path=program_path, name=name)
                title = f"Program {program_title} (learning cost {cost})"
                slug = title_to_slug(f"Program {name} (learning cost {cost})")
                toc.append(f"    - [`{name}`](#{slug})")
                contents.append(f"\n### {title}")
                contents.append(f"\n```python\n{add_line_numbers(program_info['source'])}\n```")
                contents.append("\n| Cost  | Taxon | Location |")
                contents.append("|" + "----|" * 3)
                items = sorted(
                    program_info["taxa"].items(),
                    key=lambda x: f"~{x[0]}" if x[0].startswith("meta/") else x[0],
                )
                for (taxon_name, spans) in items:
                    if taxon_name in self.hidden_taxa:
                        continue
                    taxon_cost = self.assess.taxon_cost(taxon_name)
                    s = spans_to_html(", ".join(map(couple_to_string, spans)))
                    contents.append(f"| {taxon_cost} | `{taxon_name}` | {s} |")
                contents.append("\n---")

        def programs_to_html(description: str, programs: ProgramPaths) -> str:
            details = []
            if programs:
                description = description.replace(" 1 programs", " 1 program")
                details = ["<ol>"]
                for program in sorted(programs):
                    details.append(f"  <li><code>{program}</code></li>")
                details.append("</ol>")
            return (
                "<details>\n"
                + f"  <summary>{description}.</summary>\n  "
                + "\n  ".join(details)
                + "\n</details>\n"
            )

        n = len(self.db_programs)
        summary: List[str] = ["\n# Summary"]
        summary.append(programs_to_html(f"{n} initially", list(self.db_programs)))
        for (i, operation, removed_programs) in self.result:
            n -= len(removed_programs)
            title = f"{n} remaining after operation {i} ({operation}) has filtered out {len(removed_programs)} programs"
            summary.append(programs_to_html(title, removed_programs))

        return "\n".join(toc + contents + summary)

Methods

def __init__(self,
db: JsonTagDatabase,
base_path: Union[pathlib.Path, NoneType] = None,
title_format: str = '`{name}`',
**kwargs
) ‑> NoneType

Initialize a recommendation system for the given database of programs tagged with taxa.

Args

db : JsonTagDatabase

A Python dictionary containing the JSON structure constructed by TagDatabase, i.e. the result of the parsing, labelling and taxon-ifying of a set of programs (example). The execution relies on the fields programs, taxa, importations and exportations, but ignores the field labels.

base_path : Optional[Path], optional

A path, accessible from any optional shell command under the name "{base_path}". Defaults to None.

title_format : str

a string containing a substring {name} to be interpolated in the final report with the relevant program's name. Defaults to "\`"`.

**kwargs

May include the keyword argument assessment_strategy, transmitted to LearningCostAssessor.

Warning

db is modified by side-effect. After initialization, the taxa of some programs have been augmented with the taxa of those they import, associated with an empty list of spans. Exception: the taxa starting with "meta/" are not imported.

Expand source code Browse GitHub

def __init__(
    self,
    db: JsonTagDatabase,
    base_path: Optional[Path] = None,
    title_format: str = "`{name}`",
    **kwargs,
) -> None:
    self.base_path = base_path
    self.title_format = title_format

    # copy locally some attributes and methods of a ProgramFilter instance
    program_filter = ProgramFilter(db)
    self.db_programs = program_filter.db_programs
    self.selected_programs = program_filter.selected_programs
    self.imparted_knowledge = program_filter.imparted_knowledge
    self.hidden_taxa = program_filter.hidden_taxa
    self.hidden_programs = program_filter.hidden_programs
    self.update_filter = program_filter.update_filter

    self.assess = LearningCostAssessor(self.db_programs, **kwargs)
    self.result: List[Tuple[int, Operation, ProgramPaths]] = []

def run_pipeline(self,
commands: Union[List[Command], NoneType] = None
) ‑> NoneType

Evolve the state of the filter accross pipeline commands.

Description

Execute sequentially all the commands of the pipeline. For each command:

1. Retrieve the operation ("include", "exclude", "impart" or "hide", with an optional suffix " any" (implicit) or " all").
2. Retrieve the data by calling Recommendations.parse_data().
3. Update the selected programs and/or impart the associated taxa and/or mark the programs or taxa as hidden.
4. Log the newly filtered out programs in self.result.

Finally, compute the learning costs of the remaining selected programs based on the imparted knowledge.

Args

commands : Optional[List[Command]], optional

A list of dictionaries with two mandatory entries:

• "operation": a string among "include", "exclude", "impart" and "hide", with an optional suffix " any" (implicit) or " all".
• "data": either a string consisting in a shell command, or an heterogeneous list of criteria: patterns (matching either program paths or taxon names) or semantic triples (of the form subject, predicate, object).

Defaults to None, which is treated as an empty list.

Expand source code Browse GitHub

def run_pipeline(self, commands: Optional[List[Command]] = None) -> None:
    commands = commands or []
    if not isinstance(commands, list):
        print_warning(f"commands are ignored (should be a list).")
        commands = []
    current = set(self.db_programs)
    print(f"\nProcessing {len(commands)} commands on {len(current)} programs.")

    # Execute sequentially all the commands of the pipeline
    for (i, command) in enumerate(commands, 1):

        # Retrieve the operation
        try:
            operation = command["operation"]
        except KeyError:
            print_warning(f"operation {i} is ignored (not specified).")
            continue
        except TypeError:
            print_warning(f"operation {i} is ignored (malformed).")
            continue
        (operation, n) = regex.subn(" all", "", operation)
        quantifier = "all" if n == 1 else "any"
        operation = Operation(operation.replace(" any", ""))
        if operation not in ("include", "exclude", "impart", "hide"):
            print_warning(f"operation {i} ({operation}) is ignored (unknown).")
            continue

        # Retrieve the data
        try:
            data = self.parse_data(command.get("data", []), operation)
        except MalformedData:
            print_warning(f"unable to interpret the data of operation {i} ({operation}).")
            continue
        if not data:
            print_warning(f"operation {i} ({operation}) is ignored (no data).")
            continue

        # Update the state of the filter: selected / hidden programs, imparted / hidden taxa
        self.update_filter(data, operation, quantifier)

        # Log the statistics of the filter state for the last operation
        (previous, current) = (current, set(self.selected_programs))
        self.result.append((i, operation, sorted(previous - current)))
        print(f"  {len(current)} programs remaining after operation {i} ({operation}).")

    self.assess.set_imparted_knowledge(self.imparted_knowledge)
    self.assessed_programs = self.assess(self.selected_programs)

def parse_data(self,
data: Union[str, List[Union[str, Tuple[str, str, str]]]],
operation: Operation
) ‑> List[Union[str, Tuple[str, str, str]]]

Retrieve the list of criteria to which to apply the given operation.

Description

In case data is a list of criteria, return it unchanged. Otherwise, if it is a string, execute it as a shell command, capture its output and return it as a list of strings. Otherwise, fail.

Note

A shell command cannot produce semantic triples. Its output is necessarily interpreted as a list of patterns.

Args

data : Union[str, List[Criterion]]

Either a shell command (str) or a list of criteria. A criterion is either a pattern (str) or a semantic triple (Tuple[str, str, str]) subject, predicate, object.

operation : Operation

Either "include", "exclude", "impart" or "hide" (note that the optional suffix " all" has been previously suppressed).

Raises

MalformedData

Indicates that the field "data" of a pipeline command cannot be interpreted.

Returns

List[Criterion]

A list of criteria.

Expand source code Browse GitHub

def parse_data(
    self,
    data: Union[str, List[Criterion]],
    operation: Operation,
) -> List[Criterion]:
    if isinstance(data, list):  # The object can either be a list of criteria...
        return data
    elif isinstance(data, str):  # ... or a shell command printing them on stdout
        criteria = subprocess.run(
            str(data).format(base_path=self.base_path),  # str() needed by mypy
            stdout=subprocess.PIPE,  # From Python 3.7, these two arguments can be
            stderr=subprocess.PIPE,  # ... replaced by: capture_output=True
            encoding="utf-8",
            shell=True,
            check=True,
        )
        return [criterion for criterion in criteria.stdout.strip().split("\n")]
    else:
        raise MalformedData

def get_markdown(self,
span_column_width: int = 30,
sorting_strategy: str = 'by_cost_and_sloc',
grouping_strategy: str = 'by_cost_bucket'
) ‑> str

Iterate on the results. Group, sort and output them.

Expand source code Browse GitHub

def get_markdown(
    self,
    span_column_width: int = 30,
    sorting_strategy: str = "by_cost_and_sloc",
    grouping_strategy: str = "by_cost_bucket",
) -> str:
    # Define some helper functions.

    title_to_slug = title_to_slug_factory()
    spans_to_html = enumeration_to_txt_factory(span_column_width, "_imported_")
    display_count = lambda n: f"{n} program" + ("" if n == 1 else "s")

    sorting_key = None
    if sorting_strategy == "by_cost_and_sloc":
        sorting_key = lambda x: (x[0], len(self.db_programs[x[1]]["source"].split("\n")))
    elif sorting_strategy == "lexicographic":
        sorting_key = lambda x: x[1]

    grouping_key = lambda x: "(default: no group)"
    if grouping_strategy == "by_cost_bucket":
        grouping_key = cost_bucket

    # Group resulting programs into cost buckets, and sort each group by increasing difficulty.

    toc_data: Dict[str, AssessedPrograms] = defaultdict(list)
    for (cost, program_path) in self.assessed_programs:
        if program_path in self.hidden_programs:
            continue
        toc_data[grouping_key(cost)].append((cost, program_path))
    for costs_and_program_paths in toc_data.values():
        costs_and_program_paths.sort(key=sorting_key)

    # Accumulate simultaneously the TOC and the contents.

    toc: List[str] = ["# Table of contents"]
    contents: List[str] = ["# Recommended programs"]

    for (bounds, costs_and_program_paths) in toc_data.items():

        title = f"{display_count(len(costs_and_program_paths))} of learning cost {bounds}"
        toc.append(f"- [`{title}`](#{title_to_slug(title)})")
        contents.append(f"\n## {title}")

        for (cost, program_path) in costs_and_program_paths:
            program_info = self.db_programs[program_path]
            name = regex.sub(r"(?:.+/)?(.+)", r"\1", program_path)  # last segment of the path
            program_title = self.title_format.format(path=program_path, name=name)
            title = f"Program {program_title} (learning cost {cost})"
            slug = title_to_slug(f"Program {name} (learning cost {cost})")
            toc.append(f"    - [`{name}`](#{slug})")
            contents.append(f"\n### {title}")
            contents.append(f"\n```python\n{add_line_numbers(program_info['source'])}\n```")
            contents.append("\n| Cost  | Taxon | Location |")
            contents.append("|" + "----|" * 3)
            items = sorted(
                program_info["taxa"].items(),
                key=lambda x: f"~{x[0]}" if x[0].startswith("meta/") else x[0],
            )
            for (taxon_name, spans) in items:
                if taxon_name in self.hidden_taxa:
                    continue
                taxon_cost = self.assess.taxon_cost(taxon_name)
                s = spans_to_html(", ".join(map(couple_to_string, spans)))
                contents.append(f"| {taxon_cost} | `{taxon_name}` | {s} |")
            contents.append("\n---")

    def programs_to_html(description: str, programs: ProgramPaths) -> str:
        details = []
        if programs:
            description = description.replace(" 1 programs", " 1 program")
            details = ["<ol>"]
            for program in sorted(programs):
                details.append(f"  <li><code>{program}</code></li>")
            details.append("</ol>")
        return (
            "<details>\n"
            + f"  <summary>{description}.</summary>\n  "
            + "\n  ".join(details)
            + "\n</details>\n"
        )

    n = len(self.db_programs)
    summary: List[str] = ["\n# Summary"]
    summary.append(programs_to_html(f"{n} initially", list(self.db_programs)))
    for (i, operation, removed_programs) in self.result:
        n -= len(removed_programs)
        title = f"{n} remaining after operation {i} ({operation}) has filtered out {len(removed_programs)} programs"
        summary.append(programs_to_html(title, removed_programs))

    return "\n".join(toc + contents + summary)