{
  "run_id": "ablation_20260601_050013",
  "created_at": "2026-06-01T05:00:13.683693+00:00",
  "selected_runs": [
    "full_hybrid_graph_memory",
    "no_memory",
    "vector_only_memory",
    "graphrag_only_memory",
    "llm_only_baseline"
  ],
  "output_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013",
  "dry_run": false,
  "runs": [
    {
      "name": "full_hybrid_graph_memory",
      "purpose": "Current best system: hybrid vector + GraphRAG retrieval, Neo4j timeline memory, full planning/edit/repair stack.",
      "kind": "pipeline",
      "command": [
        "/Users/carlosrortiz/Documents/csc583-CosineOfThrones/ragthrones/myenv/bin/python",
        "-m",
        "ragthrones.scripts.rewrite_season8_with_memory",
        "--retrieval-mode",
        "hybrid_graph",
        "--topk",
        "24",
        "--output-root",
        "ragthrones/eval/season8_ablations/ablation_20260601_050013/full_hybrid_graph_memory",
        "--memory-project-id",
        "alt_s8_ablation_20260601_050013_full_hybrid_graph_memory",
        "--memory-target",
        "cloud",
        "--timeline-title",
        "Season 8 Ablation: full_hybrid_graph_memory",
        "--start-episode",
        "1",
        "--end-episode",
        "6",
        "--max-season-repair-attempts",
        "6",
        "--score-nss",
        "--plot-board",
        "--plot-board-html",
        "--plot-board-graph"
      ],
      "expected_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013/full_hybrid_graph_memory"
    },
    {
      "name": "no_memory",
      "purpose": "Test whether Neo4j timeline memory improves NSS and continuity.",
      "kind": "pipeline",
      "command": [
        "/Users/carlosrortiz/Documents/csc583-CosineOfThrones/ragthrones/myenv/bin/python",
        "-m",
        "ragthrones.scripts.rewrite_season8_with_memory",
        "--retrieval-mode",
        "hybrid_graph",
        "--topk",
        "24",
        "--output-root",
        "ragthrones/eval/season8_ablations/ablation_20260601_050013/no_memory",
        "--memory-project-id",
        "alt_s8_ablation_20260601_050013_no_memory",
        "--memory-target",
        "cloud",
        "--timeline-title",
        "Season 8 Ablation: no_memory",
        "--start-episode",
        "1",
        "--end-episode",
        "6",
        "--max-season-repair-attempts",
        "6",
        "--score-nss",
        "--plot-board",
        "--plot-board-html",
        "--plot-board-graph",
        "--no-memory"
      ],
      "expected_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013/no_memory"
    },
    {
      "name": "vector_only_memory",
      "purpose": "Test whether graph retrieval improves over vector retrieval with memory held on.",
      "kind": "pipeline",
      "command": [
        "/Users/carlosrortiz/Documents/csc583-CosineOfThrones/ragthrones/myenv/bin/python",
        "-m",
        "ragthrones.scripts.rewrite_season8_with_memory",
        "--retrieval-mode",
        "vector",
        "--topk",
        "24",
        "--output-root",
        "ragthrones/eval/season8_ablations/ablation_20260601_050013/vector_only_memory",
        "--memory-project-id",
        "alt_s8_ablation_20260601_050013_vector_only_memory",
        "--memory-target",
        "cloud",
        "--timeline-title",
        "Season 8 Ablation: vector_only_memory",
        "--start-episode",
        "1",
        "--end-episode",
        "6",
        "--max-season-repair-attempts",
        "6",
        "--score-nss",
        "--plot-board",
        "--plot-board-html",
        "--plot-board-graph"
      ],
      "expected_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013/vector_only_memory"
    },
    {
      "name": "graphrag_only_memory",
      "purpose": "Test whether GraphRAG alone is sufficient with memory held on.",
      "kind": "pipeline",
      "command": [
        "/Users/carlosrortiz/Documents/csc583-CosineOfThrones/ragthrones/myenv/bin/python",
        "-m",
        "ragthrones.scripts.rewrite_season8_with_memory",
        "--retrieval-mode",
        "graphrag",
        "--topk",
        "24",
        "--output-root",
        "ragthrones/eval/season8_ablations/ablation_20260601_050013/graphrag_only_memory",
        "--memory-project-id",
        "alt_s8_ablation_20260601_050013_graphrag_only_memory",
        "--memory-target",
        "cloud",
        "--timeline-title",
        "Season 8 Ablation: graphrag_only_memory",
        "--start-episode",
        "1",
        "--end-episode",
        "6",
        "--max-season-repair-attempts",
        "6",
        "--score-nss",
        "--plot-board",
        "--plot-board-html",
        "--plot-board-graph"
      ],
      "expected_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013/graphrag_only_memory"
    },
    {
      "name": "llm_only_baseline",
      "purpose": "Native LLM baseline without retrieval, memory, planning, storyboard, editor, or season repair.",
      "kind": "baseline",
      "command": [
        "/Users/carlosrortiz/Documents/csc583-CosineOfThrones/ragthrones/myenv/bin/python",
        "-m",
        "ragthrones.scripts.rewrite_season8_llm_baseline",
        "--output-root",
        "ragthrones/eval/season8_ablations/ablation_20260601_050013/baseline",
        "--run-id",
        "llm_only_baseline",
        "--timeline-title",
        "Season 8 Ablation: llm_only_baseline",
        "--start-episode",
        "1",
        "--end-episode",
        "6",
        "--score-nss"
      ],
      "expected_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013/baseline"
    }
  ]
}