""" CaP-X arXiv paper — figure build script. Regenerates all 8 publication figures as PDFs in this directory. Usage ----- python3 figures-build.py [--only N] Each figure is implemented in its own ``def make_figN()`` function. Constraints ----------- * matplotlib + numpy + pandas + scipy only (no seaborn/plotly). * Wilson 95 % CI is computed analytically inline (no scipy.stats.beta). * DPI >= 200, vectorised PDF. """ from __future__ import annotations import math import os import sys from pathlib import Path import matplotlib as mpl import matplotlib.pyplot as plt import numpy as np import pandas as pd from matplotlib.colors import LogNorm from matplotlib.patches import FancyArrowPatch, FancyBboxPatch # --------------------------------------------------------------------------- # Paths # --------------------------------------------------------------------------- REPO = Path("/Users/ryeol-13/GitHub/oh-my-cap-x") ANALYSIS = REPO / "outputs" / "analysis" OUTDIR = REPO / "docs" / "paper" / "figures" OUTDIR.mkdir(parents=True, exist_ok=True) # --------------------------------------------------------------------------- # Style # --------------------------------------------------------------------------- try: plt.style.use("seaborn-v0_8-paper") except OSError: plt.style.use("seaborn-paper") mpl.rcParams.update( { "savefig.dpi": 300, "figure.dpi": 200, "pdf.fonttype": 42, "ps.fonttype": 42, "axes.labelsize": 9, "axes.titlesize": 9, "xtick.labelsize": 8, "ytick.labelsize": 8, "legend.fontsize": 8, "font.family": "DejaVu Sans", "axes.spines.top": False, "axes.spines.right": False, } ) # Color-blind friendly tab10 palette TAB10 = plt.get_cmap("tab10").colors C_BLUE = TAB10[0] C_ORANGE = TAB10[1] C_GREEN = TAB10[2] C_RED = TAB10[3] C_PURPLE = TAB10[4] C_BROWN = TAB10[5] C_GREY = "#666666" # --------------------------------------------------------------------------- # Wilson 95% CI (analytic, no scipy.stats) # --------------------------------------------------------------------------- def wilson_ci(successes: int, n: int, z: float = 1.96) -> tuple[float, float, float]: """Return (point estimate, lower, upper) for binomial proportion.""" if n == 0: return (0.0, 0.0, 0.0) p = successes / n z2 = z * z denom = 1.0 + z2 / n centre = (p + z2 / (2 * n)) / denom half = (z * math.sqrt((p * (1 - p) / n) + (z2 / (4 * n * n)))) / denom return (p, max(0.0, centre - half), min(1.0, centre + half)) def save(fig, name: str) -> Path: out = OUTDIR / name fig.savefig(out, format="pdf", bbox_inches="tight") # Website companions: GitHub Pages cannot reliably thumbnail PDFs in cards, # so every paper figure also gets a PNG sibling for the project website. fig.savefig(out.with_suffix(".png"), format="png", dpi=220, bbox_inches="tight") plt.close(fig) return out # --------------------------------------------------------------------------- # Figure 1 — skill usage matrix heatmap # --------------------------------------------------------------------------- def make_fig1(): csv = ANALYSIS / "phase_1_1" / "matrix_trial_x_skill.csv" df = pd.read_csv(csv) skill_cols = [ "execute_grasp", "lift_object", "plan_best_grasp_on_mask", "transform_cam_to_world", "move_to_pose_world", "pose_matrix_to_pos_quat", "pixel_mask_to_world_points", "get_grasp_pose_for_mask", "rotmat_to_quat_wxyz", "select_top_grasp", "grasp_pose_to_ik", ] short_labels = [ "exec_grasp", "lift_obj", "best_grasp", "cam2world", "move_pose", "pose2quat", "mask2pts", "grasp_pose", "rot2quat", "select_top", "grasp_IK", ] M = df[skill_cols].to_numpy(dtype=float) # (89, 11) row_order = np.argsort(-M.sum(axis=1)) M = M[row_order] M_clipped = np.clip(M, 0, 10) fig = plt.figure(figsize=(3.55, 4.25), constrained_layout=True) gs = fig.add_gridspec(1, 2, width_ratios=[24, 1.3], wspace=0.05) ax = fig.add_subplot(gs[0, 0]) cax = fig.add_subplot(gs[0, 1]) cmap = plt.get_cmap("viridis").copy() cmap.set_under("#f5f5f5") im = ax.imshow( M_clipped, aspect="auto", cmap=cmap, norm=LogNorm(vmin=1, vmax=10), interpolation="nearest", ) ax.set_xticks(range(len(skill_cols))) ax.set_xticklabels(short_labels, rotation=55, ha="right", rotation_mode="anchor", fontsize=6.2) ax.tick_params(axis="x", pad=1) ax.set_ylabel(f"Trial directories (n = {M.shape[0]})", labelpad=4) ax.set_yticks([0, M.shape[0] - 1]) ax.set_yticklabels(["1", str(M.shape[0])]) # Mark the two dead skills without sending labels into the colorbar area. dead_idxs = [skill_cols.index("select_top_grasp"), skill_cols.index("grasp_pose_to_ik")] for idx in dead_idxs: ax.add_patch( plt.Rectangle( (idx - 0.5, -0.5), 1, M.shape[0], fill=False, edgecolor=C_RED, linewidth=1.0, linestyle="--", ) ) ax.text( np.mean(dead_idxs), M.shape[0] * 0.52, "dead\nskills\n0 calls", fontsize=6.1, color=C_RED, ha="center", va="center", bbox=dict(boxstyle="round,pad=0.18", fc="white", ec="none", alpha=0.9), ) cbar = fig.colorbar(im, cax=cax) cbar.set_label("calls/trial\nlog scale", fontsize=6.2, labelpad=2) cbar.ax.tick_params(labelsize=6.2, pad=1) return save(fig, "fig-skill-usage-matrix.pdf") # --------------------------------------------------------------------------- # Figure 2 — unpromoted pool composition + gate failure overlay # --------------------------------------------------------------------------- def make_fig2(): df = pd.read_csv(ANALYSIS / "phase_1_2" / "unpromoted_classified.csv") total = len(df) n_junk = int((df["genuine_junk"] == 1).sum()) n_near = int((df["near_miss"] == 1).sum()) n_dup = int((df["promoted_hash_match"] == 1).sum()) n_other = total - n_junk - n_near - n_dup fail_gen = int((df["fail_generality"] == 1).sum()) / total fail_exec = int((df["fail_execution"] == 1).sum()) / total fail_doc = int((df["fail_docstring"] == 1).sum()) / total fail_cx = int((df["fail_complexity"] == 1).sum()) / total fig = plt.figure(figsize=(3.55, 2.65), constrained_layout=True) gs = fig.add_gridspec(1, 2, width_ratios=[1.0, 1.1], wspace=0.18) ax1 = fig.add_subplot(gs[0, 0]) ax2 = fig.add_subplot(gs[0, 1]) cats = ["junk", "near-miss", "dup", "other"] counts = [n_junk, n_near, n_dup, n_other] colors = [C_GREY, C_ORANGE, C_PURPLE, C_BLUE] bottom = 0 for c, n, col in zip(cats, counts, colors): ax1.bar([0], [n], bottom=bottom, color=col, edgecolor="white", linewidth=0.7, label=f"{c}: {n} ({n/total*100:.0f}%)") if n / total > 0.06: ax1.text(0, bottom + n / 2, f"{n}", ha="center", va="center", fontsize=7, color="white", weight="bold") bottom += n ax1.set_title("Unpromoted pool", fontsize=8.5, pad=4) ax1.set_xlim(-0.55, 0.55) ax1.set_xticks([]) ax1.set_ylim(0, total) ax1.set_ylabel(f"skills (n = {total})") ax1.legend(loc="upper center", bbox_to_anchor=(0.5, -0.04), frameon=False, fontsize=6.1, ncol=1, handlelength=0.9, borderaxespad=0.0) gate_names = ["generality", "execution", "docstring", "complexity"] gate_vals = [fail_gen, fail_exec, fail_doc, fail_cx] gate_colors = [C_RED, C_ORANGE, TAB10[8], C_GREEN] ypos = np.arange(len(gate_names)) ax2.barh(ypos, [v * 100 for v in gate_vals], color=gate_colors, edgecolor="white", linewidth=0.6) for y, v in zip(ypos, gate_vals): xval = v * 100 ha = "right" if xval > 12 else "left" xpos = xval - 3 if xval > 12 else xval + 3 color = "white" if xval > 45 else "black" ax2.text(xpos, y, f"{xval:.0f}%", va="center", ha=ha, fontsize=7, color=color, weight="bold" if xval > 45 else "normal") ax2.set_title("Gate failures", fontsize=8.5, pad=4) ax2.set_yticks(ypos) ax2.set_yticklabels(gate_names, fontsize=7) ax2.invert_yaxis() ax2.set_xlim(0, 100) ax2.set_xlabel("fail rate (%)", fontsize=7.5) ax2.set_xticks([0, 50, 100]) ax2.grid(True, axis="x", lw=0.25, alpha=0.35) return save(fig, "fig-unpromoted-pool.pdf") # --------------------------------------------------------------------------- # Figure 3 — ablation bar with Wilson CIs # --------------------------------------------------------------------------- def make_fig3(): items = [ ("empty_ns", 13, 30, C_GREY), ("P21_a", 39, 50, C_PURPLE), ("C1", 67, 70, C_BLUE), ("C2", 68, 70, C_BLUE), ("C3v2", 58, 70, C_RED), ("manual_11", 66, 70, C_GREEN), ] labels, ks, ns, colors = zip(*items) pts, los, his = zip(*[wilson_ci(k, n) for k, n in zip(ks, ns)]) err_lo = [p - lo for p, lo in zip(pts, los)] err_hi = [hi - p for p, hi in zip(pts, his)] fig, ax = plt.subplots(figsize=(3.55, 2.75), constrained_layout=True) x = np.arange(len(items)) ax.bar(x, [p * 100 for p in pts], yerr=[np.array(err_lo) * 100, np.array(err_hi) * 100], color=colors, edgecolor="white", linewidth=0.5, capsize=2.5, error_kw=dict(lw=0.8, ecolor="black")) for xi, k, n, hi in zip(x, ks, ns, his): ax.text(xi, min(106, hi * 100 + 2.0), f"{k}/{n}", ha="center", va="bottom", fontsize=6.7, bbox=dict(boxstyle="round,pad=0.05", fc="white", ec="none", alpha=0.72)) ax.set_xticks(x) ax.set_xticklabels(["empty\nns", "P21_a", "C1", "C2", "C3v2", "manual\n11"], rotation=0, ha="center", fontsize=7.4) ax.set_ylabel("Success rate (%)") ax.set_ylim(0, 112) ax.set_yticks([0, 25, 50, 75, 100]) ax.axhline(100, color="black", lw=0.4, alpha=0.3) upper = wilson_ci(39, 50)[2] * 100 ax.axhline(upper, color=C_PURPLE, lw=0.7, alpha=0.5, ls="--") ax.text(len(items) - 0.25, upper + 1.0, "P21_a CI upper", color=C_PURPLE, fontsize=6.2, ha="right", va="bottom", bbox=dict(boxstyle="round,pad=0.08", fc="white", ec="none", alpha=0.8)) ax.grid(True, axis="y", lw=0.25, alpha=0.35) return save(fig, "fig-ablation-bar.pdf") # --------------------------------------------------------------------------- # Figure 4 — C3v2 n=15 vs n=50 Wilson CI # --------------------------------------------------------------------------- def make_fig4(): pts = [(15, 14, 15, "n=15"), (50, 40, 50, "n=50"), (70, 58, 70, "n=70")] fig, ax = plt.subplots(figsize=(3.55, 2.55), constrained_layout=True) xs = [0, 1, 2] colors = [C_ORANGE, C_GREEN, C_GREEN] for xi, (n, k, _, _), col in zip(xs, pts, colors): p, lo, hi = wilson_ci(k, n) ax.errorbar([xi], [p * 100], yerr=[[(p - lo) * 100], [(hi - p) * 100]], fmt="o", capsize=4, ms=6, lw=1.2, color=col, ecolor=col) ax.text(xi + 0.07, p * 100 + (1.6 if n != 15 else 0), f"{k}/{n}\n{p*100:.1f}%", ha="left", va="center", fontsize=6.8, bbox=dict(boxstyle="round,pad=0.12", fc="white", ec="none", alpha=0.75)) ax.plot(xs, [wilson_ci(k, n)[0] * 100 for n, k, _, _ in pts], color=C_GREY, lw=0.7, alpha=0.55, zorder=0) ax.set_xticks(xs) ax.set_xticklabels(["n=15", "n=50", "n=70"]) ax.set_xlim(-0.45, 2.55) ax.set_xlabel("C3v2 sample size") ax.set_ylabel("Success rate (%)") ax.set_ylim(55, 105) ax.set_yticks([60, 70, 80, 90, 100]) ax.grid(True, axis="y", lw=0.25, alpha=0.4) return save(fig, "fig-c3v2-n50-wilson.pdf") # --------------------------------------------------------------------------- # Figure 5 — retry abstraction-shift downgrade # --------------------------------------------------------------------------- def make_fig5(): df = pd.read_csv(ANALYSIS / "phase_1_9" / "retry_classification.csv") order = ["SAME_SET", "REPLACE", "EXPAND", "SHRINK"] ks, ns = [], [] for c in order: sub = df[df["category"] == c] ns.append(len(sub)) ks.append(int(sub["success"].sum())) pts, los, his = zip(*[wilson_ci(k, n) for k, n in zip(ks, ns)]) err_lo = [p - lo for p, lo in zip(pts, los)] err_hi = [hi - p for p, hi in zip(pts, his)] fig, ax = plt.subplots(figsize=(3.55, 2.65), constrained_layout=True) x = np.arange(len(order)) colors = [C_BLUE, C_GREEN, C_ORANGE, C_RED] ax.bar(x, [p * 100 for p in pts], yerr=[np.array(err_lo) * 100, np.array(err_hi) * 100], color=colors, edgecolor="white", linewidth=0.5, capsize=3, error_kw=dict(lw=0.8, ecolor="black")) for xi, k, n, hi in zip(x, ks, ns, his): ax.text(xi, min(105, hi * 100 + 2.0), f"{k}/{n}", ha="center", va="bottom", fontsize=6.8, bbox=dict(boxstyle="round,pad=0.05", fc="white", ec="none", alpha=0.7)) ax.set_xticks(x) ax.set_xticklabels([f"{c.replace('_', ' ')}\n(n={n})" for c, n in zip(order, ns)], fontsize=7.0) ax.set_ylabel("Success rate (%)") ax.set_ylim(0, 112) ax.set_yticks([0, 25, 50, 75, 100]) ax.grid(True, axis="y", lw=0.25, alpha=0.35) return save(fig, "fig-retry-abstraction-downgrade.pdf") # --------------------------------------------------------------------------- # Figure 6 — three-fix narrative rewrite (waterfall) # --------------------------------------------------------------------------- def make_fig6(): stages = [ ("C0", 33.3), ("C1", 95.7), ("C2", 97.1), ("C3v2", 82.9), ("manual_11", 94.3), ] labels, vals = zip(*stages) deltas = [vals[i + 1] - vals[i] for i in range(len(vals) - 1)] fix_names = ["namespace\nseeding", "quality\ngates", "dedup\ntax", "smart\nselection"] fig, ax = plt.subplots(figsize=(3.55, 3.05), constrained_layout=True) x = np.arange(len(stages)) ax.plot(x, vals, "-o", color=C_BLUE, ms=4.8, lw=1.25, zorder=3) for xi, v in zip(x, vals): va = "top" if v > 90 else "bottom" off = -5 if v > 90 else 4 ax.text(xi, v + off, f"{v:.1f}%", ha="center", va=va, fontsize=7.2, weight="bold", color=C_BLUE, bbox=dict(boxstyle="round,pad=0.06", fc="white", ec="none", alpha=0.68)) band_ys = [132, 116, 132, 116] delta_colors = [C_GREEN, C_GREY, C_RED, C_GREEN] for i, (d, fname, col, by) in enumerate(zip(deltas, fix_names, delta_colors, band_ys)): x_mid = (x[i] + x[i + 1]) / 2 sign = "+" if d > 0 else "" y_seg = (vals[i] + vals[i + 1]) / 2 ax.annotate(f"{fname}\n{sign}{d:.1f} pp", xy=(x_mid, y_seg), xytext=(x_mid, by), ha="center", va="center", fontsize=6.3, color=col, bbox=dict(boxstyle="round,pad=0.12", fc="white", ec=col, lw=0.35, alpha=0.9), arrowprops=dict(arrowstyle="-", color=col, lw=0.55, alpha=0.55)) ax.set_xticks(x) ax.set_xticklabels(["C0", "C1", "C2", "C3v2", "manual\n11"], fontsize=7.8) ax.set_ylim(0, 145) ax.set_yticks([0, 25, 50, 75, 100]) ax.set_ylabel("Success rate (%)") ax.spines["left"].set_bounds(0, 100) ax.grid(True, axis="y", lw=0.25, alpha=0.35) return save(fig, "fig-3fix-narrative-rewrite.pdf") # --------------------------------------------------------------------------- # Figure 7 — cube_stack failure modes # --------------------------------------------------------------------------- def make_fig7(): """phase_2_4d directory does not exist; reconstruct from phase_2_4c (cubestack rows).""" df = pd.read_csv(ANALYSIS / "phase_2_4c" / "condition_summary.csv") cs = df[df["condition"].str.startswith("cubestack_")].copy() cs["short"] = cs["condition"].str.replace("cubestack_", "", regex=False) order = ["p22_a", "c2", "c3", "c3v2"] cs = cs.set_index("short").reindex(order).reset_index() blocks = cs["avg_code_blocks"].to_numpy(dtype=float) regens = cs["avg_regens"].to_numpy(dtype=float) succ = cs["successes"].to_numpy(dtype=int) n = cs["unique_trials"].to_numpy(dtype=int) fig = plt.figure(figsize=(3.55, 3.25), constrained_layout=True) gs = fig.add_gridspec(2, 1, height_ratios=[1.25, 0.9], hspace=0.08) ax1 = fig.add_subplot(gs[0, 0]) ax2 = fig.add_subplot(gs[1, 0], sharex=ax1) x = np.arange(len(order)) w = 0.34 ax1.bar(x - w / 2, blocks, w, color=C_BLUE, edgecolor="white", lw=0.5, label="avg code blocks") ax1.bar(x + w / 2, regens, w, color=C_ORANGE, edgecolor="white", lw=0.5, label="avg regens") ax1.set_ylabel("avg per trial", fontsize=8) ax1.set_ylim(0, max(blocks.max(), regens.max()) * 1.32) ax1.legend(loc="upper right", fontsize=6.6, frameon=False, ncol=1) ax1.grid(True, axis="y", lw=0.25, alpha=0.35) ax1.tick_params(labelbottom=False) bars = ax2.bar(x, succ, 0.5, color=C_GREEN, edgecolor="white", lw=0.5) ax2.set_ylabel("successes", fontsize=8) ax2.set_ylim(0, max(3, succ.max() + 1)) ax2.set_yticks([0, 1, 2, 3]) ax2.set_xticks(x) ax2.set_xticklabels(order, fontsize=8) ax2.grid(True, axis="y", lw=0.25, alpha=0.35) for xi, s, ni in zip(x, succ, n): ax2.text(xi, s + 0.08, f"{s}/{ni}", ha="center", va="bottom", fontsize=7.0, color=C_GREEN, weight="bold") ax2.text(0.98, 0.82, "only C3v2\nclears floor", transform=ax2.transAxes, ha="right", va="top", fontsize=6.7, color=C_GREEN, bbox=dict(boxstyle="round,pad=0.16", fc="white", ec=C_GREEN, lw=0.35, alpha=0.9)) return save(fig, "fig-cubestack-failure-modes.pdf") # --------------------------------------------------------------------------- # Figure 8 — skill library architecture (block diagram) # --------------------------------------------------------------------------- def make_fig8(): fig, ax = plt.subplots(figsize=(3.55, 4.25), constrained_layout=True) ax.set_xlim(0, 112) ax.set_ylim(0, 120) ax.axis("off") def box(x, y, w, h, text, color="#e8eef9", edge=C_BLUE, fontsize=7.5, weight="normal"): p = FancyBboxPatch((x, y), w, h, boxstyle="round,pad=0.22,rounding_size=1.5", linewidth=0.9, edgecolor=edge, facecolor=color) ax.add_patch(p) ax.text(x + w / 2, y + h / 2, text, ha="center", va="center", fontsize=fontsize, weight=weight, wrap=True) def arrow(x1, y1, x2, y2, color=C_GREY, rad=0.0): ax.add_patch(FancyArrowPatch((x1, y1), (x2, y2), arrowstyle="->", mutation_scale=10, lw=0.9, color=color, connectionstyle=f"arc3,rad={rad}")) cx, w, h = 12, 76, 10 ys = [102, 87, 72, 53, 36, 21, 6] box(cx, ys[0], w, h, "LLM trials\n89 trial directories", color="#fff4d6", edge=C_ORANGE, weight="bold") box(cx, ys[1], w, h, "AST extraction\nnamed functions + dependencies", color="#e8eef9", edge=C_BLUE) box(cx, ys[2], w, h, "341 candidates\noccurrence-counted pool", color="#e8eef9", edge=C_BLUE) # Gate block has explicit soft/hard lanes but keeps labels short enough for one column. gate_y, gate_h = ys[3], 13 box(cx, gate_y, w, gate_h, "", color="#fdeeee", edge=C_RED) ax.text(50, gate_y + gate_h - 2.1, "Quality gates", ha="center", va="center", fontsize=7.4, color=C_RED, weight="bold") ax.text(31, gate_y + 4.9, "soft:\nexec · generality · code", ha="center", va="center", fontsize=6.4, color=C_RED) ax.text(69, gate_y + 4.9, "hard:\ndeps · no-op · vision", ha="center", va="center", fontsize=6.4, color=C_RED) ax.plot([50, 50], [gate_y + 1.6, gate_y + gate_h - 4.0], color=C_RED, lw=0.35, alpha=0.5) box(cx, ys[4], w, h, "148 soft-promoted\nthen hard-fail + dedup filter", color="#e8f6e8", edge=C_GREEN) box(cx, ys[5], w, h, "dedup v2\nsignature cluster + doc-prefer survivor", color="#fff4d6", edge=C_ORANGE, fontsize=6.9) box(cx, ys[6], w, h, "11 final skills\ninjected into next LLM trial", color="#e8f6e8", edge=C_GREEN, fontsize=7.6, weight="bold") centers = [(50, y) for y in [ys[0], ys[1], ys[2], gate_y, ys[4], ys[5], ys[6]]] bottoms = [y for y in [ys[0], ys[1], ys[2], gate_y, ys[4], ys[5]]] tops = [y + h for y in [ys[1], ys[2], gate_y, ys[4], ys[5], ys[6]]] for y1, y2 in zip(bottoms, tops): arrow(50, y1, 50, y2) # Feedback loop, drawn outside the boxes to avoid covering labels. arrow(cx + w + 14, ys[6] + h / 2, cx + w + 14, ys[0] + h / 2, color=C_GREY, rad=-0.18) ax.text(cx + w + 18, 58, "reinject", rotation=90, fontsize=6.7, color=C_GREY, ha="center", va="center", style="italic") return save(fig, "fig-skill-library-architecture.pdf") # --------------------------------------------------------------------------- # Driver # --------------------------------------------------------------------------- ALL = [ make_fig1, make_fig2, make_fig3, make_fig4, make_fig5, make_fig6, make_fig7, make_fig8, ] def main(): only = None if "--only" in sys.argv: only = int(sys.argv[sys.argv.index("--only") + 1]) for i, fn in enumerate(ALL, start=1): if only is not None and i != only: continue out = fn() size = out.stat().st_size print(f"[fig {i}] {out.name:42s} {size/1024:6.1f} KB") if __name__ == "__main__": main()