analysis

The breakdown behind hbb2obb-analyze. The guide describes each table and has a worked example.

from pathlib import Path

from hbb2obb.analysis import collect_pairs, plot_analysis, render_markdown, summarise

pairs = collect_pairs(Path("data/labels_obb_gt"), Path("data/labels_obb"), hbb_dir=Path("data/labels_hbb"))
summary = summarise(pairs)
Path("analysis.md").write_text(render_markdown(summary))
plot_analysis(summary, Path("analysis.png"))

Collecting pairs

collect_pairs

collect_pairs(gt_dir: Path, pred_dir: Path, hbb_dir: Optional[Path] = None, image_sizes: Optional[Dict[str, Tuple[int, int]]] = None, excluded_classes: Sequence[int] = (), iou_threshold: float = 0.1, class_agnostic: bool = False, no_bar: bool = False) -> List[Pair]

Match every converted box to its ground truth and record what each cut needs.

Parameters:

Name	Type	Description	Default
gt_dir	Path	Directory of ground-truth OBB annotations	required
pred_dir	Path	Directory of converted OBB annotations	required
hbb_dir	Optional[Path]	Directory of the horizontal boxes the conversion was prompted with, for the identity baseline; without it that comparison is left out	None
image_sizes	Optional[Dict[str, Tuple[int, int]]]	Frame stem to (width, height), for the frame-edge cut; without it that cut is left out	None
excluded_classes	Sequence[int]	Class ids to drop before matching	()
iou_threshold	float	Minimum IoU for a match, as in hbb2obb-eval	0.1
class_agnostic	bool	Match regardless of class label	False
no_bar	bool	Disable the progress bar	False

Returns:

Type	Description
List[Pair]	One Pair per matched box.

Source code in hbb2obb/analysis.py

def collect_pairs(
    gt_dir: Path,
    pred_dir: Path,
    hbb_dir: Optional[Path] = None,
    image_sizes: Optional[Dict[str, Tuple[int, int]]] = None,
    excluded_classes: Sequence[int] = (),
    iou_threshold: float = 0.1,
    class_agnostic: bool = False,
    no_bar: bool = False,
) -> List[Pair]:
    """
    Match every converted box to its ground truth and record what each cut needs.

    Args:
        gt_dir: Directory of ground-truth OBB annotations
        pred_dir: Directory of converted OBB annotations
        hbb_dir: Directory of the horizontal boxes the conversion was prompted with, for the
                 identity baseline; without it that comparison is left out
        image_sizes: Frame stem to (width, height), for the frame-edge cut; without it that cut
                 is left out
        excluded_classes: Class ids to drop before matching
        iou_threshold: Minimum IoU for a match, as in ``hbb2obb-eval``
        class_agnostic: Match regardless of class label
        no_bar: Disable the progress bar

    Returns:
        One Pair per matched box.
    """
    gt_files = sorted(Path(gt_dir).glob("*.txt"))
    if not gt_files:
        raise SystemExit(f"no ground-truth .txt files in {gt_dir}")

    pairs: List[Pair] = []
    warned_difficult = False
    for gt_file in tqdm.tqdm(gt_files, desc="Analyzing files", leave=True, disable=no_bar):
        pred_file = Path(pred_dir) / gt_file.name
        if not pred_file.exists():
            print(f"Warning: no converted file for {gt_file.name}")
            continue

        gt_boxes = [b for b in parse_obb_file(gt_file) if b["label"] not in excluded_classes]
        pred_boxes = [b for b in parse_obb_file(pred_file) if b["label"] not in excluded_classes]
        matches, _, _ = match_boxes(gt_boxes, pred_boxes, iou_threshold, class_agnostic)

        # The flag is stored per row of the DOTA file, so it can only be attached when that file
        # holds the same boxes in the same order as the ground truth being scored.
        flags = read_difficult(gt_file.with_suffix(".dota"))
        if flags and len(flags) != len(gt_boxes) and not warned_difficult:
            print(
                f"Warning: {gt_file.with_suffix('.dota').name} has {len(flags)} rows, not {len(gt_boxes)}; "
                "the difficult flag is left out"
            )
            warned_difficult = True
        flag_by_index = dict(enumerate(flags)) if len(flags) == len(gt_boxes) else {}
        index_of = {id(b): i for i, b in enumerate(gt_boxes)}

        identity = {}
        if hbb_dir is not None:
            hbb_file = Path(hbb_dir) / gt_file.name
            if hbb_file.exists():
                hbb_boxes = read_hbb_as_boxes(hbb_file, image_sizes, gt_file.stem)
                hbb_boxes = [b for b in hbb_boxes if b["label"] not in excluded_classes]
                for gt_box, hbb_box, _ in match_boxes(gt_boxes, hbb_boxes, iou_threshold, class_agnostic)[0]:
                    identity[id(gt_box)] = hbb_box

        size = (image_sizes or {}).get(gt_file.stem)
        for gt_box, pred_box, iou in matches:
            gt_short, gt_long = box_sides(gt_box["points"])
            pred_short, pred_long = box_sides(pred_box["points"])
            gt_area = gt_box["polygon"].area
            hbb_box = identity.get(id(gt_box))
            pairs.append(
                Pair(
                    frame=gt_file.stem,
                    cls=gt_box["label"],
                    iou=float(iou),
                    angle=orientation_error(gt_box["points"], pred_box["points"]),
                    off_axis=off_axis_angle(gt_box["points"]),
                    gt_short=gt_short,
                    gt_long=gt_long,
                    pred_short=pred_short,
                    pred_long=pred_long,
                    area_ratio=float(pred_box["polygon"].area / gt_area) if gt_area > 0 else 0.0,
                    edge=is_edge_box(gt_box, size[0], size[1]) if size else None,
                    difficult=flag_by_index.get(index_of[id(gt_box)]) if flag_by_index else None,
                    identity_iou=(
                        calculate_obb_iou(gt_box["polygon"], hbb_box["polygon"]) if hbb_box is not None else None
                    ),
                    identity_angle=(
                        orientation_error(gt_box["points"], hbb_box["points"]) if hbb_box is not None else None
                    ),
                )
            )
    return pairs

Pair dataclass

One matched ground-truth and converted box, with the properties every cut reads.

Source code in hbb2obb/analysis.py

@dataclass
class Pair:
    """One matched ground-truth and converted box, with the properties every cut reads."""

    frame: str
    cls: int
    iou: float
    angle: float
    off_axis: float
    gt_short: float
    gt_long: float
    pred_short: float
    pred_long: float
    area_ratio: float
    edge: Optional[bool] = None
    difficult: Optional[int] = None
    identity_iou: Optional[float] = None
    identity_angle: Optional[float] = None

Summarising and reporting

summarise

summarise(pairs: Sequence[Pair], label_map: Optional[dict] = None, boundary_classes: Optional[Sequence[int]] = None) -> dict

Every cut and comparison, as plain data a report or a plot can render.

Source code in hbb2obb/analysis.py

def summarise(
    pairs: Sequence[Pair], label_map: Optional[dict] = None, boundary_classes: Optional[Sequence[int]] = None
) -> dict:
    """Every cut and comparison, as plain data a report or a plot can render."""
    clean = controlled(pairs)
    summary = {
        "boxes": len(pairs),
        "has_identity": any(p.identity_iou is not None for p in pairs),
        "has_edge": any(p.edge is not None for p in pairs),
        "has_difficult": any(p.difficult is not None for p in pairs),
        "overall": group_stats(pairs),
        "orientation": [(name, group_stats(g)) for name, g in orientation_cut(pairs)],
        "orientation_controlled": [(name, group_stats(g)) for name, g in orientation_cut(clean)],
        "orientation_partition": [(name, group_stats(g)) for name, g in orientation_partition(pairs)],
        "orientation_partition_controlled": [(name, group_stats(g)) for name, g in orientation_partition(clean)],
        "controlled_boxes": len(clean),
        "size": [(name, group_stats(g)) for name, g in size_cut(pairs)],
        "klass": [(name, group_stats(g)) for name, g in class_cut(pairs, label_map)],
        "side_ratios": side_ratios(clean, boundary_classes),
        "boundary_classes": list(boundary_classes) if boundary_classes else None,
    }
    if summary["has_edge"]:
        summary["edge"] = [
            ("touches an edge", group_stats([p for p in pairs if p.edge])),
            ("interior", group_stats([p for p in pairs if p.edge is False])),
        ]
    if summary["has_difficult"]:
        summary["difficult"] = [
            ("difficult 1", group_stats([p for p in pairs if p.difficult == 1])),
            ("difficult 0", group_stats([p for p in pairs if p.difficult == 0])),
        ]
    return summary

render_markdown

render_markdown(summary: dict, title: str = 'Conversion breakdown') -> str

The whole analysis as one Markdown document.

Source code in hbb2obb/analysis.py

def render_markdown(summary: dict, title: str = "Conversion breakdown") -> str:
    """The whole analysis as one Markdown document."""
    overall = summary["overall"]
    out = [f"# {title}", ""]
    out.append(
        f"{summary['boxes']} matched boxes. Mean IoU {overall['mean_iou']:.5f} "
        f"± {overall['std_iou']:.5f} (SEM {overall['sem_iou']:.5f}), median {overall['median_iou']:.5f}, "
        f"{overall['at_high']:.1%} at or above IoU {HIGH_IOU_THRESHOLD:g}, "
        f"orientation error p90 {overall['angle_p90']:.2f} deg."
    )
    out.append("")
    out.append(
        "Two runs of the same set are the same measurement only if their SEM says so: a difference "
        "smaller than one standard error is a tie."
    )
    out.append("")
    out.append(section("By ground-truth orientation", summary["orientation"]))
    out.append(
        section(
            f"By ground-truth orientation, interior and not difficult ({summary['controlled_boxes']} boxes)",
            summary["orientation_controlled"],
        )
    )
    out.append(section("By ground-truth short side", summary["size"]))
    if "edge" in summary:
        out.append(section("By frame edge", summary["edge"]))
    if "difficult" in summary:
        out.append(section("By the annotator's difficult flag", summary["difficult"]))
    out.append(section("By class", summary["klass"]))

    if summary["has_identity"]:
        out.append("### Against doing nothing\n")
        out.append(
            "A horizontal box is already a valid oriented box, so emitting the prompt unchanged is the "
            "null conversion. This is what it scores, beside the conversion, on the same matched pairs.\n"
        )
        rows = []
        for name, stats in [("all", summary["overall"])] + summary["orientation"]:
            if not stats.get("boxes") or "identity_iou" not in stats:
                continue
            rows.append(
                [
                    name,
                    f"{stats['boxes']:d}",
                    f"{stats['identity_iou']:.4f}",
                    f"{stats['mean_iou']:.4f}",
                    f"{stats['mean_iou'] - stats['identity_iou']:+.4f}",
                    f"{stats['identity_angle_p90']:.2f}",
                    f"{stats['angle_p90']:.2f}",
                ]
            )
        out.append(
            markdown_table(
                ["", "Boxes", "IoU as-is", "IoU converted", "Gain", "Angle p90 as-is", "Angle p90 converted"],
                rows,
            )
            + "\n"
        )

    out.append("### Converted-to-reference side ratios\n")
    scope = "all classes" if not summary["boundary_classes"] else f"classes {summary['boundary_classes']}"
    out.append(
        f"Interior boxes not flagged difficult, {scope}. The long side is the control. A step confined to "
        "the short side, across a boundary the geometry crosses continuously, is a property of the "
        "reference rather than of the conversion.\n"
    )
    out.append(
        markdown_table(
            ["Reference off-axis", "Boxes", "Short side", "Long side", "Area"],
            [
                [
                    r["band"],
                    f"{r['boxes']:d}",
                    f"{r['short_ratio']:.4f}" if r["boxes"] else "",
                    f"{r['long_ratio']:.4f}" if r["boxes"] else "",
                    f"{r['area_ratio']:.4f}" if r["boxes"] else "",
                ]
                for r in summary["side_ratios"]
            ],
        )
        + "\n"
    )
    return "\n".join(out)

print_analysis

print_analysis(summary: dict) -> None

The same report on the terminal, the Markdown tables laid out in columns.

Source code in hbb2obb/analysis.py

def print_analysis(summary: dict) -> None:
    """The same report on the terminal, the Markdown tables laid out in columns."""
    block: List[List[str]] = []

    def flush() -> None:
        if not block:
            return
        widths = [max(len(row[i]) for row in block) for i in range(len(block[0]))]
        for row in block:
            cells = [row[0].ljust(widths[0])] + [c.rjust(w) for c, w in zip(row[1:], widths[1:])]
            print("  " + "   ".join(cells).rstrip())
        block.clear()

    for line in render_markdown(summary).splitlines():
        stripped = line.strip()
        if stripped.startswith("|"):
            cells = [c.strip() for c in stripped.strip("|").split("|")]
            if set("".join(cells)) <= {"-"}:
                continue
            if block and len(cells) != len(block[0]):
                flush()
            block.append(cells)
            continue
        flush()
        print(line.rstrip())
    flush()

plot_analysis

plot_analysis(summary: dict, out_path: Path) -> None

Three panels: where the average comes from, what the prompt already gave, and the side ratios.

Forces the Agg backend, as the sweep plots do, so this runs on a headless machine.

Source code in hbb2obb/analysis.py

def plot_analysis(summary: dict, out_path: Path) -> None:
    """
    Three panels: where the average comes from, what the prompt already gave, and the side ratios.

    Forces the Agg backend, as the sweep plots do, so this runs on a headless machine.
    """
    import matplotlib

    matplotlib.use("Agg")
    import matplotlib.pyplot as plt

    converted, prompt, grid = "#3b6ea5", "#c0c0c0", "#cccccc"
    panels = 3 if summary["has_identity"] else 2
    fig, axes = plt.subplots(1, panels, figsize=(5.6 * panels, 4.6))
    axes = np.atleast_1d(axes)

    def ticks(ax, names, counts):
        ax.set_xticks(range(len(names)))
        ax.set_xticklabels([f"{n}\n({c})" for n, c in zip(names, counts)], fontsize=8)
        ax.grid(axis="y", color=grid, linewidth=0.6)
        ax.set_axisbelow(True)

    named = [(n, st_) for n, st_ in summary["orientation_partition_controlled"] if st_.get("boxes")]
    labels = [n for n, _ in named]
    values = [s["mean_iou"] for _, s in named]

    ax = axes[0]
    ax.bar(labels, values, color=converted)
    ax.errorbar(labels, values, yerr=[s["sem_iou"] for _, s in named], fmt="none", ecolor="black", capsize=3)
    ax.set_ylabel("Mean IoU")
    ax.set_ylim(0, 1.05)
    ax.set_xlabel("Reference orientation, degrees off the image axes")
    ax.set_title("Where the average comes from\n(interior, not flagged difficult)")
    ticks(ax, labels, [s["boxes"] for _, s in named])

    if summary["has_identity"]:
        ax = axes[1]
        rows = [(n, s) for n, s in summary["orientation_partition"] if s.get("boxes") and "identity_iou" in s]
        x = np.arange(len(rows))
        ax.bar(x - 0.2, [s["identity_iou"] for _, s in rows], 0.4, label="prompt as-is", color=prompt)
        ax.bar(x + 0.2, [s["mean_iou"] for _, s in rows], 0.4, label="converted", color=converted)
        ax.set_ylabel("Mean IoU")
        ax.set_ylim(0, 1.05)
        ax.set_xlabel("Reference orientation, degrees off the image axes")
        ax.set_title("Against doing nothing")
        ax.legend(loc="lower left", frameon=False, fontsize=9)
        ticks(ax, [n for n, _ in rows], [s["boxes"] for _, s in rows])

    ax = axes[-1]
    rows = [r for r in summary["side_ratios"] if r["boxes"]]
    x = np.arange(len(rows))
    ax.plot(x, [r["short_ratio"] for r in rows], "o-", label="short side", color="#b5651d")
    ax.plot(x, [r["long_ratio"] for r in rows], "s-", label="long side", color=converted)
    ax.axhline(1.0, color="black", linewidth=0.8, linestyle=":")
    ax.set_ylabel("Converted / reference")
    ax.set_xlabel("Reference orientation, degrees off the image axes")
    scope = "all classes" if not summary["boundary_classes"] else f"class {summary['boundary_classes']}"
    ax.set_title(f"Side ratios by reference orientation\n(interior, not flagged difficult, {scope})")
    ax.legend(frameon=False, fontsize=9)
    ticks(
        ax,
        [r["band"].replace(" deg", "").replace("exactly aligned", "aligned") for r in rows],
        [r["boxes"] for r in rows],
    )

    fig.tight_layout()
    fig.savefig(out_path, dpi=150)
    plt.close(fig)