{ "run_id": "ablation_20260601_050013", "runs": [ { "name": "full_hybrid_graph_memory", "purpose": "Current best system: hybrid vector + GraphRAG retrieval, Neo4j timeline memory, full planning/edit/repair stack.", "kind": "pipeline", "retrieval_mode": "hybrid_graph", "memory_enabled": true, "status": "completed", "returncode": 0, "elapsed_seconds": 4657.2, "command": [ "/Users/carlosrortiz/Documents/csc583-CosineOfThrones/ragthrones/myenv/bin/python", "-m", "ragthrones.scripts.rewrite_season8_with_memory", "--retrieval-mode", "hybrid_graph", "--topk", "24", "--output-root", "ragthrones/eval/season8_ablations/ablation_20260601_050013/full_hybrid_graph_memory", "--memory-project-id", "alt_s8_ablation_20260601_050013_full_hybrid_graph_memory", "--memory-target", "cloud", "--timeline-title", "Season 8 Ablation: full_hybrid_graph_memory", "--start-episode", "1", "--end-episode", "6", "--max-season-repair-attempts", "6", "--score-nss", "--plot-board", "--plot-board-html", "--plot-board-graph" ], "expected_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013/full_hybrid_graph_memory", "artifact_dir": "ragthrones/eval/season8_ablations/ablation_20260601_050013/full_hybrid_graph_memory/timeline_e7afa18865bb4123a22efdfd93667da5", "summary_path": "ragthrones/eval/season8_ablations/ablation_20260601_050013/full_hybrid_graph_memory/timeline_e7afa18865bb4123a22efdfd93667da5/season8_summary.json", "log_path": "ragthrones/eval/season8_ablations/ablation_20260601_050013/logs/full_hybrid_graph_memory.log", "episodes_written": 6, "average_nss_total": 157.33, "season_check_passed": false, "season_check_summary": "The alternate final season of Game of Thrones presents a compelling narrative but ultimately fails to resolve several major arcs, particularly regarding the political landscape of Westeros and the fates of key characters. While the season builds tension and character development effectively, it leaves unresolved threads that impact the overall closure of the story.", "missing_resolutions": [ "The final political settlement of Westeros is not clearly defined, leaving the question of governance ambiguous.", "Cersei's fate is not adequately resolved, as her plans for revenge and influence remain unaddressed.", "The implications of Jon's parentage and his relationship with Daenerys are not fully settled, creating lingering uncertainty.", "The final status of the North and its autonomy is not explicitly stated, leaving the political landscape unclear." ], "cross_episode_contradictions": [ "The portrayal of the political tensions between Daenerys and Sansa escalates throughout the season but does not culminate in a clear resolution in Episode 6." ], "consistency_ced_total": 0.0, "consistency_error_count": 0, "consistency_counts_by_category": { "timeline_plot_logic": 0, "characterization": 0, "worldbuilding_setting": 0, "factual_detail": 0, "narrative_style": 0 }, "wall_time_seconds": 4506.39, "total_tokens": 1977870, "usage_totals": { "season_planner": { "prompt_tokens": 15654, "completion_tokens": 4860, "total_tokens": 20514 }, "episode_planner": { "prompt_tokens": 146520, "completion_tokens": 21151, "total_tokens": 167671 }, "episode_editor": { "prompt_tokens": 200783, "completion_tokens": 10717, "total_tokens": 211500 }, "episode_patch": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "continuity_editor": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "storyboard": { "prompt_tokens": 161619, "completion_tokens": 23831, "total_tokens": 185450 }, "writer": { "prompt_tokens": 690678, "completion_tokens": 82873, "total_tokens": 773551 }, "checker": { "prompt_tokens": 206230, "completion_tokens": 2509, "total_tokens": 208739 }, "nss": { "prompt_tokens": 349803, "completion_tokens": 10726, "total_tokens": 360529 }, "plot_board": { "prompt_tokens": 43324, "completion_tokens": 6592, "total_tokens": 49916 } }, "nss_by_episode": [ { "episode_number": 1, "title": "The Gathering Storm", "nss_total": 156, "consistency_errors": 0 }, { "episode_number": 2, "title": "Episode Title: \"Alliances\"", "nss_total": 156, "consistency_errors": 0 }, { "episode_number": 3, "title": "The First Confrontation", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 4, "title": "Fractured Alliances", "nss_total": 156, "consistency_errors": 0 }, { "episode_number": 5, "title": "The Last Stand", "nss_total": 156, "consistency_errors": 0 }, { "episode_number": 6, "title": "A New Dawn", "nss_total": 160, "consistency_errors": 0 } ] }, { "name": "no_memory", "purpose": "Test whether Neo4j timeline memory improves NSS and continuity.", "kind": "pipeline", "retrieval_mode": "hybrid_graph", "memory_enabled": false, "status": "completed", "returncode": 0, "elapsed_seconds": 3936.47, "command": [ "/Users/carlosrortiz/Documents/csc583-CosineOfThrones/ragthrones/myenv/bin/python", "-m", "ragthrones.scripts.rewrite_season8_with_memory", "--retrieval-mode", "hybrid_graph", "--topk", "24", "--output-root", "ragthrones/eval/season8_ablations/ablation_20260601_050013/no_memory", "--memory-project-id", "alt_s8_ablation_20260601_050013_no_memory", "--memory-target", "cloud", "--timeline-title", "Season 8 Ablation: no_memory", "--start-episode", "1", "--end-episode", "6", "--max-season-repair-attempts", "6", "--score-nss", "--plot-board", "--plot-board-html", "--plot-board-graph", "--no-memory" ], "expected_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013/no_memory", "artifact_dir": "ragthrones/eval/season8_ablations/ablation_20260601_050013/no_memory/timeline_e45352d4e75a44e598615ff50be1f510", "summary_path": "ragthrones/eval/season8_ablations/ablation_20260601_050013/no_memory/timeline_e45352d4e75a44e598615ff50be1f510/season8_summary.json", "log_path": "ragthrones/eval/season8_ablations/ablation_20260601_050013/logs/no_memory.log", "episodes_written": 6, "average_nss_total": 157.33, "season_check_passed": false, "season_check_summary": "The alternate final season of Game of Thrones presents a compelling narrative but ultimately fails to resolve several major arcs and leaves key political outcomes ambiguous. While the battle against the Night King is staged and concluded, the political landscape and character resolutions in Episode 6 do not meet the required final state criteria, particularly regarding the governance of Westeros and the fate of Cersei Lannister.", "missing_resolutions": [ "The final governance structure of Westeros is not clearly established in terms of power dynamics and acceptance by the realm.", "Cersei Lannister's fate is resolved with her exile, but the implications of her manipulations and potential future threats are not addressed.", "Jon Snow's final role and relationship with Daenerys are not fully articulated, leaving his identity and political standing ambiguous.", "The final states for Tyrion, Arya, Bran, and Jaime are not explicitly detailed, which undermines their arcs." ], "cross_episode_contradictions": [], "consistency_ced_total": 0.0, "consistency_error_count": 0, "consistency_counts_by_category": { "timeline_plot_logic": 0, "characterization": 0, "worldbuilding_setting": 0, "factual_detail": 0, "narrative_style": 0 }, "wall_time_seconds": 3862.39, "total_tokens": 1840161, "usage_totals": { "season_planner": { "prompt_tokens": 15652, "completion_tokens": 4626, "total_tokens": 20278 }, "episode_planner": { "prompt_tokens": 131514, "completion_tokens": 21430, "total_tokens": 152944 }, "episode_editor": { "prompt_tokens": 155405, "completion_tokens": 9203, "total_tokens": 164608 }, "episode_patch": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "continuity_editor": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "storyboard": { "prompt_tokens": 151030, "completion_tokens": 23232, "total_tokens": 174262 }, "writer": { "prompt_tokens": 658644, "completion_tokens": 82998, "total_tokens": 741642 }, "checker": { "prompt_tokens": 176708, "completion_tokens": 2461, "total_tokens": 179169 }, "nss": { "prompt_tokens": 350238, "completion_tokens": 10789, "total_tokens": 361027 }, "plot_board": { "prompt_tokens": 41473, "completion_tokens": 4758, "total_tokens": 46231 } }, "nss_by_episode": [ { "episode_number": 1, "title": "The Gathering Storm", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 2, "title": "The Gathering Storm", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 3, "title": "The Long Night's Reckoning", "nss_total": 148, "consistency_errors": 0 }, { "episode_number": 4, "title": "The Weight of the Crown", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 5, "title": "The Final Confrontation", "nss_total": 156, "consistency_errors": 0 }, { "episode_number": 6, "title": "A New Dawn", "nss_total": 160, "consistency_errors": 0 } ] }, { "name": "vector_only_memory", "purpose": "Test whether graph retrieval improves over vector retrieval with memory held on.", "kind": "pipeline", "retrieval_mode": "vector", "memory_enabled": true, "status": "completed", "returncode": 0, "elapsed_seconds": 4412.48, "command": [ "/Users/carlosrortiz/Documents/csc583-CosineOfThrones/ragthrones/myenv/bin/python", "-m", "ragthrones.scripts.rewrite_season8_with_memory", "--retrieval-mode", "vector", "--topk", "24", "--output-root", "ragthrones/eval/season8_ablations/ablation_20260601_050013/vector_only_memory", "--memory-project-id", "alt_s8_ablation_20260601_050013_vector_only_memory", "--memory-target", "cloud", "--timeline-title", "Season 8 Ablation: vector_only_memory", "--start-episode", "1", "--end-episode", "6", "--max-season-repair-attempts", "6", "--score-nss", "--plot-board", "--plot-board-html", "--plot-board-graph" ], "expected_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013/vector_only_memory", "artifact_dir": "ragthrones/eval/season8_ablations/ablation_20260601_050013/vector_only_memory/timeline_d87565ff4fcb4a65ab18a6b40ee98465", "summary_path": "ragthrones/eval/season8_ablations/ablation_20260601_050013/vector_only_memory/timeline_d87565ff4fcb4a65ab18a6b40ee98465/season8_summary.json", "log_path": "ragthrones/eval/season8_ablations/ablation_20260601_050013/logs/vector_only_memory.log", "episodes_written": 6, "average_nss_total": 156.83, "season_check_passed": false, "season_check_summary": "The alternate final season of Game of Thrones presents a coherent narrative with significant character arcs and political developments. However, it fails to resolve key elements, particularly regarding the final governance of Westeros and the fate of Cersei Lannister, leaving lingering questions about the political landscape and potential future conflicts.", "missing_resolutions": [ "The final governance structure of Westeros is not clearly defined in Episode 6, leaving the realm's acceptance of Daenerys and Jon's rule ambiguous.", "Cersei's fate is not fully resolved, as her loyalists are left with potential for future plots and unrest.", "The political ramifications of the battle and the future governance of Westeros are not adequately addressed, creating uncertainty." ], "cross_episode_contradictions": [], "consistency_ced_total": 0.0, "consistency_error_count": 0, "consistency_counts_by_category": { "timeline_plot_logic": 0, "characterization": 0, "worldbuilding_setting": 0, "factual_detail": 0, "narrative_style": 0 }, "wall_time_seconds": 4357.35, "total_tokens": 1833996, "usage_totals": { "season_planner": { "prompt_tokens": 23250, "completion_tokens": 4801, "total_tokens": 28051 }, "episode_planner": { "prompt_tokens": 171303, "completion_tokens": 19242, "total_tokens": 190545 }, "episode_editor": { "prompt_tokens": 161572, "completion_tokens": 8291, "total_tokens": 169863 }, "episode_patch": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "continuity_editor": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "storyboard": { "prompt_tokens": 185339, "completion_tokens": 21236, "total_tokens": 206575 }, "writer": { "prompt_tokens": 743326, "completion_tokens": 77365, "total_tokens": 820691 }, "checker": { "prompt_tokens": 202184, "completion_tokens": 2327, "total_tokens": 204511 }, "nss": { "prompt_tokens": 157910, "completion_tokens": 10105, "total_tokens": 168015 }, "plot_board": { "prompt_tokens": 42342, "completion_tokens": 3403, "total_tokens": 45745 } }, "nss_by_episode": [ { "episode_number": 1, "title": "Winter's Tension", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 2, "title": "The Gathering Storm", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 3, "title": "The Long Night", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 4, "title": "The Weight of the Crown", "nss_total": 145, "consistency_errors": 0 }, { "episode_number": 5, "title": "The Last Stand", "nss_total": 156, "consistency_errors": 0 }, { "episode_number": 6, "title": "A New Dawn", "nss_total": 160, "consistency_errors": 0 } ] }, { "name": "graphrag_only_memory", "purpose": "Test whether GraphRAG alone is sufficient with memory held on.", "kind": "pipeline", "retrieval_mode": "graphrag", "memory_enabled": true, "status": "completed", "returncode": 0, "elapsed_seconds": 3853.63, "command": [ "/Users/carlosrortiz/Documents/csc583-CosineOfThrones/ragthrones/myenv/bin/python", "-m", "ragthrones.scripts.rewrite_season8_with_memory", "--retrieval-mode", "graphrag", "--topk", "24", "--output-root", "ragthrones/eval/season8_ablations/ablation_20260601_050013/graphrag_only_memory", "--memory-project-id", "alt_s8_ablation_20260601_050013_graphrag_only_memory", "--memory-target", "cloud", "--timeline-title", "Season 8 Ablation: graphrag_only_memory", "--start-episode", "1", "--end-episode", "6", "--max-season-repair-attempts", "6", "--score-nss", "--plot-board", "--plot-board-html", "--plot-board-graph" ], "expected_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013/graphrag_only_memory", "artifact_dir": "ragthrones/eval/season8_ablations/ablation_20260601_050013/graphrag_only_memory/timeline_1c53902b36784c4a90dbcb247359c39b", "summary_path": "ragthrones/eval/season8_ablations/ablation_20260601_050013/graphrag_only_memory/timeline_1c53902b36784c4a90dbcb247359c39b/season8_summary.json", "log_path": "ragthrones/eval/season8_ablations/ablation_20260601_050013/logs/graphrag_only_memory.log", "episodes_written": 6, "average_nss_total": 159.33, "season_check_passed": false, "season_check_summary": "The alternate final season of Game of Thrones presents a compelling narrative but ultimately fails to resolve several major arcs and leaves key political outcomes ambiguous. While the character developments and plot twists are engaging, the finale does not provide a clear and definitive conclusion to the overarching conflicts, particularly regarding Jon's parentage, the governance of Westeros, and the fate of Cersei Lannister.", "missing_resolutions": [ "Jon's parentage is still not revealed to the wider realm, leaving potential political implications unresolved.", "The rift between Sansa and Daenerys remains, requiring further resolution in the aftermath of the battle.", "The final political settlement of Westeros lacks clarity, particularly regarding the acceptance of Daenerys's rule and the council's effectiveness." ], "cross_episode_contradictions": [], "consistency_ced_total": 1.388, "consistency_error_count": 1, "consistency_counts_by_category": { "timeline_plot_logic": 1, "characterization": 0, "worldbuilding_setting": 0, "factual_detail": 0, "narrative_style": 0 }, "wall_time_seconds": 3784.9, "total_tokens": 1958808, "usage_totals": { "season_planner": { "prompt_tokens": 16566, "completion_tokens": 5768, "total_tokens": 22334 }, "episode_planner": { "prompt_tokens": 145303, "completion_tokens": 21000, "total_tokens": 166303 }, "episode_editor": { "prompt_tokens": 198217, "completion_tokens": 10618, "total_tokens": 208835 }, "episode_patch": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "continuity_editor": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "storyboard": { "prompt_tokens": 159527, "completion_tokens": 22433, "total_tokens": 181960 }, "writer": { "prompt_tokens": 678757, "completion_tokens": 80443, "total_tokens": 759200 }, "checker": { "prompt_tokens": 203231, "completion_tokens": 2411, "total_tokens": 205642 }, "nss": { "prompt_tokens": 357469, "completion_tokens": 10675, "total_tokens": 368144 }, "plot_board": { "prompt_tokens": 41756, "completion_tokens": 4634, "total_tokens": 46390 } }, "nss_by_episode": [ { "episode_number": 1, "title": "The Gathering Storm", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 2, "title": "The Calm Before the Storm", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 3, "title": "The First Confrontation", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 4, "title": "The Weight of the Crown", "nss_total": 160, "consistency_errors": 0 }, { "episode_number": 5, "title": "The Final Confrontation", "nss_total": 156, "consistency_errors": 0 }, { "episode_number": 6, "title": "A New Dawn", "nss_total": 160, "consistency_errors": 1 } ] }, { "name": "llm_only_baseline", "purpose": "Native LLM baseline without retrieval, memory, planning, storyboard, editor, or season repair.", "kind": "baseline", "retrieval_mode": null, "memory_enabled": false, "status": "completed", "returncode": 0, "elapsed_seconds": 215.23, "command": [ "/Users/carlosrortiz/Documents/csc583-CosineOfThrones/ragthrones/myenv/bin/python", "-m", "ragthrones.scripts.rewrite_season8_llm_baseline", "--output-root", "ragthrones/eval/season8_ablations/ablation_20260601_050013/baseline", "--run-id", "llm_only_baseline", "--timeline-title", "Season 8 Ablation: llm_only_baseline", "--start-episode", "1", "--end-episode", "6", "--score-nss" ], "expected_root": "ragthrones/eval/season8_ablations/ablation_20260601_050013/baseline", "artifact_dir": "ragthrones/eval/season8_ablations/ablation_20260601_050013/baseline/llm_only_baseline", "summary_path": "ragthrones/eval/season8_ablations/ablation_20260601_050013/baseline/llm_only_baseline/season8_summary.json", "log_path": "ragthrones/eval/season8_ablations/ablation_20260601_050013/logs/llm_only_baseline.log", "episodes_written": 6, "average_nss_total": 141.33, "season_check_passed": null, "season_check_summary": null, "missing_resolutions": [], "cross_episode_contradictions": [], "consistency_ced_total": null, "consistency_error_count": null, "consistency_counts_by_category": {}, "wall_time_seconds": 214.02, "total_tokens": 33635, "usage_totals": { "writer": { "prompt_tokens": 5180, "completion_tokens": 8149, "total_tokens": 13329 }, "nss": { "prompt_tokens": 16484, "completion_tokens": 3822, "total_tokens": 20306 } }, "nss_by_episode": [ { "episode_number": 1, "title": "Episode 1: \"The Gathering Storm\"", "nss_total": 132, "consistency_errors": null }, { "episode_number": 2, "title": "Episode 2: \"The Weight of Ice\"", "nss_total": 156, "consistency_errors": null }, { "episode_number": 3, "title": "Episode 3: \"The Last Stand\"", "nss_total": 130, "consistency_errors": null }, { "episode_number": 4, "title": "Episode 4: \"The Cost of War\"", "nss_total": 146, "consistency_errors": null }, { "episode_number": 5, "title": "Episode 5: \"The Iron Price\"", "nss_total": 124, "consistency_errors": null }, { "episode_number": 6, "title": "Episode 6: \"A Song of Ice and Fire\"", "nss_total": 160, "consistency_errors": null } ] } ] }