{
  "snapshot_id": "deepseek-v02-release",
  "benchmark_contract": "netopsbench-0.2-release",
  "agent": "minimal-deepagent",
  "provider": "deepseek",
  "model": "deepseek-v4-pro",
  "thinking": false,
  "temperature": 0,
  "trace_publication_status": "published_huggingface",
  "trace_dataset_url": "https://huggingface.co/datasets/yyyyyt/netopsbench-trace/tree/main/releases/netopsbench-0.2",
  "trace_dataset_commit": "f7ef8e5ecde5eba030c9b4cd9ec4cda7703edc42",
  "workers_per_scale": 2,
  "started_at": "2026-07-29T06:27:23.885216+00:00",
  "completed_at": "2026-07-29T11:52:58.154903+00:00",
  "scale_registry_sha256": "c18ef08a3e3d5a6940cae679c42a378942a5305b8b9dc8a495006c136203f35c",
  "aggregate": {
    "total_cases": 109,
    "positive_cases": 96,
    "healthy_cases": 13,
    "primary_reward": 0.729358,
    "correct_verdict": 88,
    "verdict_accuracy": 0.807339,
    "positive_detected": 75,
    "positive_detection_rate": 0.78125,
    "healthy_correct": 13,
    "healthy_accuracy": 1,
    "correct_device": 69,
    "device_localization_rate": 0.71875,
    "device_localization_given_detected": 0.92,
    "interface_applicable_cases": 56,
    "correct_interface": 26,
    "interface_localization_rate": 0.464286,
    "correct_fault_type": 43,
    "fault_type_accuracy": 0.447917,
    "inconclusive": 13,
    "false_healthy_on_positive": 8,
    "case_valid": 109,
    "coverage_complete": 109,
    "cleanup_success": 109,
    "local_probe_errors": 0,
    "total_input_tokens": 35209730,
    "total_output_tokens": 356879,
    "total_tool_calls": 2600,
    "total_agent_seconds": 8447.710028,
    "avg_input_tokens_per_case": 323025.046,
    "avg_output_tokens_per_case": 3274.119,
    "avg_tool_calls_per_case": 23.853,
    "avg_agent_seconds_per_case": 77.502
  },
  "scales": {
    "xs": {
      "run_id": "run-20260729T063132Z",
      "cases": 14,
      "positive_cases": 12,
      "healthy_cases": 2,
      "primary_reward": 0.75,
      "detection_f1": 0.957,
      "device_localization_rate": 0.75,
      "interface_localization_rate": 0.571,
      "localization_composite_score": 0.66,
      "fault_type_accuracy": 0.417,
      "avg_agent_seconds": 61.3,
      "avg_tool_calls": 22.6,
      "avg_input_tokens": 174275.1,
      "avg_output_tokens": 2677.8,
      "trace_count": 14,
      "trace_error_count": 0,
      "scale_profile_sha256": "483287175c5a06ca2c32b37ff65d1224d12dcd05bd606164125beef197f3605c",
      "trace_archive": "runs/deepseek/deepseek-v4-pro/netopsbench-0.2/xs/run-20260729T063132Z.tar.zst"
    },
    "small": {
      "run_id": "run-20260729T070853Z",
      "cases": 15,
      "positive_cases": 12,
      "healthy_cases": 3,
      "primary_reward": 0.833,
      "detection_f1": 0.957,
      "device_localization_rate": 0.833,
      "interface_localization_rate": 0.571,
      "localization_composite_score": 0.702,
      "fault_type_accuracy": 0.583,
      "avg_agent_seconds": 65.1,
      "avg_tool_calls": 20.9,
      "avg_input_tokens": 221433.3,
      "avg_output_tokens": 2876.9,
      "trace_count": 15,
      "trace_error_count": 0,
      "scale_profile_sha256": "f2755e1d35ae4d848cecd4c3a96daa3cfcbac472db10c8d697d04de3bdb7bb63",
      "trace_archive": "runs/deepseek/deepseek-v4-pro/netopsbench-0.2/small/run-20260729T070853Z.tar.zst"
    },
    "medium": {
      "run_id": "run-20260729T075032Z",
      "cases": 28,
      "positive_cases": 24,
      "healthy_cases": 4,
      "primary_reward": 0.714,
      "detection_f1": 0.857,
      "device_localization_rate": 0.708,
      "interface_localization_rate": 0.429,
      "localization_composite_score": 0.569,
      "fault_type_accuracy": 0.375,
      "avg_agent_seconds": 79.6,
      "avg_tool_calls": 23.5,
      "avg_input_tokens": 298350,
      "avg_output_tokens": 3332.1,
      "trace_count": 28,
      "trace_error_count": 4,
      "scale_profile_sha256": "438738df59b17cf07993016cd4d3694fbabac995d81b62ec46b6c60d3f6d5ca6",
      "trace_archive": "runs/deepseek/deepseek-v4-pro/netopsbench-0.2/medium/run-20260729T075032Z.tar.zst"
    },
    "large": {
      "run_id": "run-20260729T091138Z",
      "cases": 52,
      "positive_cases": 48,
      "healthy_cases": 4,
      "primary_reward": 0.702,
      "detection_f1": 0.843,
      "device_localization_rate": 0.688,
      "interface_localization_rate": 0.429,
      "localization_composite_score": 0.558,
      "fault_type_accuracy": 0.458,
      "avg_agent_seconds": 84.3,
      "avg_tool_calls": 25.2,
      "avg_input_tokens": 405665,
      "avg_output_tokens": 3518,
      "trace_count": 52,
      "trace_error_count": 9,
      "scale_profile_sha256": "056af355f7fd02c0266bde91e2ee0d6633a7287a063770c67c4914b92f228370",
      "trace_archive": "runs/deepseek/deepseek-v4-pro/netopsbench-0.2/large/run-20260729T091138Z.tar.zst"
    }
  },
  "large_topologies": {
    "xlarge": {
      "snapshot_kind": "retry_adjusted_composite",
      "benchmark_contract": "netopsbench-0.2-release",
      "catalog_cases": 70,
      "operationally_valid_cases": 70,
      "agent_scored_cases": 70,
      "positive_cases": 66,
      "healthy_cases": 4,
      "primary_reward": 0.564286,
      "correct_verdict": 49,
      "verdict_accuracy": 0.7,
      "detection_f1": 0.810798,
      "positive_detected": 45,
      "positive_detection_rate": 0.6818,
      "healthy_correct": 4,
      "correct_device": 37,
      "device_localization_rate": 0.5606,
      "device_localization_given_detected": 0.8222,
      "interface_applicable_cases": 40,
      "correct_interface": 12,
      "interface_localization_rate": 0.3,
      "correct_fault_type": 29,
      "fault_type_accuracy": 0.4394,
      "inconclusive": 7,
      "avg_agent_seconds": 97.531,
      "avg_tool_calls": 22.6,
      "avg_input_tokens": 727848.5,
      "avg_output_tokens": 3233.8,
      "coverage_complete": 70,
      "cleanup_success": 70,
      "local_probe_error_cases": 0,
      "trace_count": 70,
      "trace_archive": "runs/deepseek/deepseek-v4-pro/netopsbench-0.2/xlarge/composite-20260728T165136Z.tar.zst",
      "source_runs": [
        "run-20260727T192408Z",
        "run-20260728T093348Z",
        "run-20260728T154049Z",
        "run-20260728T161048Z"
      ]
    },
    "fat-tree-k8": {
      "snapshot_kind": "resumed_composite",
      "benchmark_contract": "netopsbench-0.2-release",
      "catalog_cases": 70,
      "operationally_valid_cases": 70,
      "agent_scored_cases": 70,
      "positive_cases": 66,
      "healthy_cases": 4,
      "primary_reward": 0.635714,
      "correct_verdict": 48,
      "verdict_accuracy": 0.685714,
      "detection_f1": 0.8,
      "positive_detected": 44,
      "positive_detection_rate": 0.666667,
      "healthy_correct": 4,
      "correct_device": 42,
      "device_localization_rate": 0.636364,
      "device_localization_given_detected": 0.954545,
      "interface_applicable_cases": 40,
      "correct_interface": 18,
      "interface_localization_rate": 0.45,
      "correct_fault_type": 23,
      "fault_type_accuracy": 0.348485,
      "inconclusive": 14,
      "avg_agent_seconds": 102.913,
      "avg_tool_calls": 26.343,
      "avg_input_tokens": 594658.857,
      "avg_output_tokens": 3687.257,
      "coverage_complete": 70,
      "cleanup_success": 70,
      "local_probe_error_cases": 0,
      "trace_count": 70,
      "trace_archive": "runs/deepseek/deepseek-v4-pro/netopsbench-0.2/fat-tree-k8/composite-fat-tree-k8-20260730T082347Z.tar.zst",
      "source_runs": [
        "run-20260729T173713Z",
        "run-20260730T033230Z"
      ],
      "publication_note": "The first run produced 16 complete results before its interactive launcher ended. The resumed run evaluated the remaining 54 scenarios on the same healthy two-worker runtime. Two unfinished orphan traces from the interrupted pair are excluded."
    },
    "fat-tree-k12": {
      "snapshot_kind": "retry_adjusted_composite",
      "benchmark_contract": "netopsbench-0.2-release",
      "catalog_cases": 70,
      "operationally_valid_cases": 70,
      "agent_scored_cases": 70,
      "positive_cases": 66,
      "healthy_cases": 4,
      "primary_reward": 0.55,
      "correct_verdict": 42,
      "verdict_accuracy": 0.6,
      "detection_f1": 0.731,
      "positive_detected": 38,
      "positive_detection_rate": 0.575758,
      "healthy_correct": 4,
      "correct_device": 37,
      "device_localization_rate": 0.561,
      "device_localization_given_detected": 0.973684,
      "interface_applicable_cases": 40,
      "correct_interface": 11,
      "interface_localization_rate": 0.275,
      "correct_fault_type": 20,
      "fault_type_accuracy": 0.303,
      "avg_agent_seconds": 102.9,
      "avg_tool_calls": 25.9,
      "avg_input_tokens": 784323.6,
      "avg_output_tokens": 3761.5,
      "coverage_complete": 70,
      "cleanup_success": 70,
      "local_probe_error_cases": 0,
      "trace_count": 70,
      "trace_archive": "runs/deepseek/deepseek-v4-pro/netopsbench-0.2/fat-tree-k12/composite-fat-tree-k12-v020.tar.zst",
      "source_runs": [
        "run-20260727T053519Z",
        "run-20260727T093625Z",
        "run-20260727T162451Z",
        "run-20260813T041401Z"
      ],
      "publication_note": "All 70 cases are operationally valid and have schema-valid Agent results and ATIF trajectories."
    }
  },
  "trace_inventory": {
    "xs_through_large": 109,
    "xlarge": 70,
    "fat_tree_k8": 70,
    "fat_tree_k12": 70,
    "total": 319,
    "format": "ATIF v1.7",
    "scoring_sidecars": 319
  },
  "fault_families": {
    "acl_misconfig": {
      "cases": 8,
      "correct_verdict": 7,
      "correct_device": 7,
      "correct_fault_type": 2,
      "inconclusive": 1,
      "pingmesh_signal_cases": 8,
      "pingmesh_anomalies": 580
    },
    "bgp_neighbor_misconfig": {
      "cases": 8,
      "correct_verdict": 8,
      "correct_device": 8,
      "correct_fault_type": 8,
      "inconclusive": 0,
      "pingmesh_signal_cases": 0,
      "pingmesh_anomalies": 0
    },
    "blackhole_route": {
      "cases": 8,
      "correct_verdict": 8,
      "correct_device": 8,
      "correct_fault_type": 8,
      "inconclusive": 0,
      "pingmesh_signal_cases": 8,
      "pingmesh_anomalies": 46
    },
    "device_down": {
      "cases": 8,
      "correct_verdict": 8,
      "correct_device": 8,
      "correct_fault_type": 8,
      "inconclusive": 0,
      "pingmesh_signal_cases": 8,
      "pingmesh_anomalies": 2112
    },
    "healthy_network": {
      "cases": 13,
      "correct_verdict": 13,
      "inconclusive": 0,
      "pingmesh_signal_cases": 0,
      "pingmesh_anomalies": 0
    },
    "high_latency": {
      "cases": 8,
      "correct_verdict": 2,
      "correct_device": 1,
      "correct_fault_type": 0,
      "inconclusive": 2,
      "pingmesh_signal_cases": 8,
      "pingmesh_anomalies": 1500
    },
    "link_down": {
      "cases": 8,
      "correct_verdict": 8,
      "correct_device": 8,
      "correct_fault_type": 8,
      "inconclusive": 0,
      "pingmesh_signal_cases": 8,
      "pingmesh_anomalies": 580
    },
    "link_flapping": {
      "cases": 8,
      "correct_verdict": 8,
      "correct_device": 8,
      "correct_fault_type": 0,
      "inconclusive": 0,
      "pingmesh_signal_cases": 7,
      "pingmesh_anomalies": 79
    },
    "mtu_mismatch": {
      "cases": 8,
      "correct_verdict": 4,
      "correct_device": 3,
      "correct_fault_type": 3,
      "inconclusive": 4,
      "pingmesh_signal_cases": 8,
      "pingmesh_anomalies": 1488
    },
    "packet_corruption": {
      "cases": 8,
      "correct_verdict": 4,
      "correct_device": 2,
      "correct_fault_type": 0,
      "inconclusive": 1,
      "pingmesh_signal_cases": 8,
      "pingmesh_anomalies": 221
    },
    "packet_loss": {
      "cases": 8,
      "correct_verdict": 3,
      "correct_device": 2,
      "correct_fault_type": 1,
      "inconclusive": 4,
      "pingmesh_signal_cases": 8,
      "pingmesh_anomalies": 352
    },
    "route_policy_misconfig": {
      "cases": 8,
      "correct_verdict": 7,
      "correct_device": 7,
      "correct_fault_type": 0,
      "inconclusive": 1,
      "pingmesh_signal_cases": 8,
      "pingmesh_anomalies": 550
    },
    "static_route_misconfig": {
      "cases": 8,
      "correct_verdict": 8,
      "correct_device": 7,
      "correct_fault_type": 5,
      "inconclusive": 0,
      "pingmesh_signal_cases": 8,
      "pingmesh_anomalies": 46
    }
  },
  "representative_traces": [
    {
      "scenario_id": "generated_healthy_network_large_001",
      "trace_id": "run-20260729T091138Z:worker-1:case-f4b8b8630661:8cacca5effa8",
      "result": "correct_healthy",
      "pingmesh_anomalies": 0
    },
    {
      "scenario_id": "generated_device_down_large_001",
      "trace_id": "run-20260729T091138Z:worker-1:case-080adf30c6ea:40af722a80e4",
      "result": "correct_device_down",
      "pingmesh_anomalies": 492
    },
    {
      "scenario_id": "generated_bgp_neighbor_misconfig_large_001",
      "trace_id": "run-20260729T091138Z:worker-1:case-3b43309ba2bc:3006351e1391",
      "result": "correct_control_plane_diagnosis_without_pingmesh_anomaly",
      "pingmesh_anomalies": 0
    },
    {
      "scenario_id": "generated_acl_misconfig_large_001",
      "trace_id": "run-20260729T091138Z:worker-1:case-996d0dc09129:8f54416796a8",
      "result": "correct_acl_diagnosis",
      "pingmesh_anomalies": 126
    },
    {
      "scenario_id": "generated_high_latency_large_004",
      "trace_id": "run-20260729T091138Z:worker-2:case-f1124ac4716d:03bf204ce996",
      "result": "false_healthy_with_clear_latency_signal",
      "pingmesh_anomalies": 365
    },
    {
      "scenario_id": "generated_mtu_mismatch_large_003",
      "trace_id": "run-20260729T091138Z:worker-1:case-24ee8bd87b38:499b485a2b02",
      "result": "inconclusive_with_clear_mtu_signal",
      "pingmesh_anomalies": 376
    },
    {
      "scenario_id": "generated_packet_loss_large_001",
      "trace_id": "run-20260729T091138Z:worker-1:case-2c3d40b194db:ab16d1d06370",
      "result": "inconclusive_with_clear_loss_signal",
      "pingmesh_anomalies": 79
    }
  ]
}