{
  "schema_version": 1,
  "study": "ISO-AdamW versus AdamW: pretrained GRPO feasibility pilot",
  "experiment_dates": [
    "2026-09-10",
    "2026-09-11"
  ],
  "artifact_scope": "Aggregate results and protocol only; not a complete reproduction package. Raw answers, training code archive, and final model weights are not included.",
  "paper": "https://arxiv.org/html/2607.19331v1#S4.SS3",
  "training_source_commit": "13fa794ef064381d222846d4c9d13720d1fae016",
  "data_sha256": "d4b2e706745698af33390db38b5eb2682a257a1c3ca66c960133fc36abab743f",
  "protocol": {
    "model": {
      "id": "Qwen/Qwen3-1.7B-Base",
      "revision": "ea980cb0a6c2ae4b936e82123acc929f1cec04c1"
    },
    "dataset": {
      "id": "openai/gsm8k",
      "revision": "740312add88f781978c0658806c59bc2815b9866",
      "split_seed": 20260910,
      "few_shot": 4,
      "calibration_size": 32,
      "train_size": 3000,
      "dev_size": 500,
      "test_size": 1000
    },
    "training": {
      "seed": 42,
      "group_size": 4,
      "prompts_per_step": 2,
      "micro_batch_size": 1,
      "kl_beta": 0.001,
      "iso_svd_driver": null,
      "iso_polar_backend": "newton_schulz",
      "qualification_steps": 3,
      "sweep_steps": 12,
      "pilot_steps": 50,
      "learning_rates": {
        "adamw": [
          5e-07,
          1.5e-06,
          3e-06
        ],
        "iso_adamw": [
          3e-07,
          7.5e-07,
          2e-06
        ]
      }
    },
    "generation": {
      "max_prompt_tokens": 2048,
      "max_completion_tokens": 512,
      "temperature": 1.0,
      "stop_string": "\nQuestion:"
    },
    "evaluation": {
      "batch_size": 8,
      "sweep_dev_size": 128,
      "curve_every": 10,
      "test_only_after_selection": true
    }
  },
  "execution": {
    "hardware": "One preemptible NVIDIA H200; arms run sequentially",
    "trainer": "TRL 1.7 GRPOTrainer with Hugging Face generation and SDPA",
    "weights_and_optimizer_state_dtype": "FP32",
    "autocast_dtype": "BF16",
    "gradient_checkpointing": true,
    "lora": false,
    "quantization": false,
    "vllm": false,
    "cuda_graphs": false,
    "weight_decay_both_arms": 0,
    "training_prompt_groups_per_pilot": 100,
    "training_completions_per_pilot": 400,
    "timing_scope": "Training wall time includes generation, backward, optimizer updates and first-step factor initialization; excludes evaluation. Completion lengths differ between arms.",
    "memory_scope": "Peak allocated CUDA memory, not total reserved memory or device utilization"
  },
  "analysis": {
    "scope": "one-seed pretrained feasibility pilot; not a paper reproduction",
    "initial_weights_sha256": "24d53e3c8bc72ba957b3b8a83d3c9ba7bf179cf2045d726152d7b6705f3956a6",
    "base_test_accuracy": 0.73,
    "test_questions": 1000,
    "arms": {
      "adamw": {
        "lr": 3e-06,
        "steps": 50,
        "test_accuracy": 0.754,
        "dev_accuracy": 0.862,
        "correct": 754.0,
        "wall_step_seconds": {
          "min": 7.290537879000112,
          "max": 22.313698420000037,
          "mean": 13.702434472460027,
          "mean_after_first": 13.7412529779388,
          "first": 11.80032770400021
        },
        "train_seconds": 685.1217236230013,
        "max_allocated_GiB": 35.78995752334595,
        "reward_last10": 0.725,
        "reward_max": 1.0,
        "mean_completion_tokens": 164.685,
        "mean_trl_non_eos_fraction": 0.085,
        "test_parsed_fraction": 0.993,
        "test_capped_fraction": 0.002,
        "test_mean_tokens": 167.135,
        "dev_curve": [
          {
            "step": 10,
            "train_seconds": 109.78773018599986,
            "accuracy": 0.84375,
            "count": 128,
            "samples": 1,
            "parsed_fraction": 0.9453125,
            "capped_fraction": 0,
            "mean_tokens": 142.203125,
            "seconds": 145.14462152500028
          },
          {
            "step": 20,
            "train_seconds": 229.285595673,
            "accuracy": 0.796875,
            "count": 128,
            "samples": 1,
            "parsed_fraction": 0.96875,
            "capped_fraction": 0,
            "mean_tokens": 147.875,
            "seconds": 149.70247170200037
          },
          {
            "step": 30,
            "train_seconds": 381.79532448899954,
            "accuracy": 0.8671875,
            "count": 128,
            "samples": 1,
            "parsed_fraction": 0.9765625,
            "capped_fraction": 0,
            "mean_tokens": 163.7421875,
            "seconds": 151.37481195100008
          },
          {
            "step": 40,
            "train_seconds": 538.1790163120013,
            "accuracy": 0.8828125,
            "count": 128,
            "samples": 1,
            "parsed_fraction": 1,
            "capped_fraction": 0,
            "mean_tokens": 158.2890625,
            "seconds": 151.196345888
          },
          {
            "step": 50,
            "train_seconds": 685.1217236230013,
            "accuracy": 0.890625,
            "count": 128,
            "samples": 1,
            "parsed_fraction": 1,
            "capped_fraction": 0,
            "mean_tokens": 156.5625,
            "seconds": 148.4357076869992
          }
        ]
      },
      "iso_adamw": {
        "lr": 7.5e-07,
        "steps": 50,
        "test_accuracy": 0.758,
        "dev_accuracy": 0.876,
        "correct": 758.0,
        "wall_step_seconds": {
          "min": 8.779025607000221,
          "max": 69.7143563970003,
          "mean": 14.556557249480012,
          "mean_after_first": 13.430887879122453,
          "first": 69.7143563970003
        },
        "train_seconds": 727.8278624740005,
        "max_allocated_GiB": 52.80185079574585,
        "reward_last10": 0.775,
        "reward_max": 1.0,
        "mean_completion_tokens": 150.69,
        "mean_trl_non_eos_fraction": 0.0425,
        "test_parsed_fraction": 0.995,
        "test_capped_fraction": 0.002,
        "test_mean_tokens": 148.174,
        "dev_curve": [
          {
            "step": 10,
            "train_seconds": 189.67801409799995,
            "accuracy": 0.8359375,
            "count": 128,
            "samples": 1,
            "parsed_fraction": 0.9609375,
            "capped_fraction": 0.0078125,
            "mean_tokens": 138.3125,
            "seconds": 144.5814460290003
          },
          {
            "step": 20,
            "train_seconds": 315.3427664369992,
            "accuracy": 0.8515625,
            "count": 128,
            "samples": 1,
            "parsed_fraction": 0.9921875,
            "capped_fraction": 0.0078125,
            "mean_tokens": 141.7265625,
            "seconds": 144.05371112800003
          },
          {
            "step": 30,
            "train_seconds": 455.9662157779994,
            "accuracy": 0.8359375,
            "count": 128,
            "samples": 1,
            "parsed_fraction": 1,
            "capped_fraction": 0,
            "mean_tokens": 139.640625,
            "seconds": 138.03691341900048
          },
          {
            "step": 40,
            "train_seconds": 601.4266597919996,
            "accuracy": 0.8828125,
            "count": 128,
            "samples": 1,
            "parsed_fraction": 1,
            "capped_fraction": 0,
            "mean_tokens": 138.7890625,
            "seconds": 136.77724565400058
          },
          {
            "step": 50,
            "train_seconds": 727.8278624740005,
            "accuracy": 0.8984375,
            "count": 128,
            "samples": 1,
            "parsed_fraction": 1,
            "capped_fraction": 0,
            "mean_tokens": 141.8515625,
            "seconds": 151.42877945100008
          }
        ]
      }
    },
    "paired_comparisons": {
      "adamw_minus_base": {
        "count": 1000,
        "accuracy_difference": 0.024,
        "paired_question_bootstrap_95": [
          -0.002,
          0.05
        ],
        "candidate_only_correct": 104,
        "reference_only_correct": 80,
        "bootstrap_seed": 20260910,
        "bootstrap_samples": 20000
      },
      "iso_adamw_minus_base": {
        "count": 1000,
        "accuracy_difference": 0.028,
        "paired_question_bootstrap_95": [
          0.001,
          0.054
        ],
        "candidate_only_correct": 108,
        "reference_only_correct": 80,
        "bootstrap_seed": 20260910,
        "bootstrap_samples": 20000
      },
      "iso_adamw_minus_adamw": {
        "count": 1000,
        "accuracy_difference": 0.004,
        "paired_question_bootstrap_95": [
          -0.018,
          0.027
        ],
        "candidate_only_correct": 68,
        "reference_only_correct": 64,
        "bootstrap_seed": 20260910,
        "bootstrap_samples": 20000
      }
    },
    "interval_scope": "Paired test-question uncertainty only; one training seed, no seed-variance estimate."
  },
  "polar_backend_qualification": {
    "driver": null,
    "backend": "newton_schulz",
    "matrices": [
      {
        "shape": [
          1024,
          1024
        ],
        "frames_per_step": 56,
        "default_seconds": 0.13633774599998105,
        "selected_seconds": 0.03400721399998474,
        "relative_difference": 2.545733094476284e-13,
        "orthogonal_error": 1.9891342162365545e-15
      },
      {
        "shape": [
          2048,
          1024
        ],
        "frames_per_step": 56,
        "default_seconds": 0.08659979199998702,
        "selected_seconds": 0.0010651520000237724,
        "relative_difference": 2.3072807880263805e-13,
        "orthogonal_error": 1.873255697732774e-15
      },
      {
        "shape": [
          2048,
          2048
        ],
        "frames_per_step": 197,
        "default_seconds": 0.5055846989999964,
        "selected_seconds": 0.002617323000009719,
        "relative_difference": 6.060536184421883e-13,
        "orthogonal_error": 2.847041895179211e-15
      },
      {
        "shape": [
          6144,
          2048
        ],
        "frames_per_step": 84,
        "default_seconds": 0.3505286109999872,
        "selected_seconds": 0.00660131700001898,
        "relative_difference": 5.492718094226293e-13,
        "orthogonal_error": 2.8456105526855354e-15
      },
      {
        "shape": [
          151936,
          2048
        ],
        "frames_per_step": 1,
        "default_seconds": 0.5844696959999851,
        "selected_seconds": 0.16237663100000077,
        "relative_difference": 5.587859716151974e-13,
        "orthogonal_error": 4.003581347711893e-15
      }
    ],
    "passed": true,
    "estimated_default_polar_seconds": 142.1135608509964,
    "estimated_selected_polar_seconds": 3.1965523860039866
  },
  "validation": {
    "focused_tests_passed": 50,
    "exact_checkpoint_2_to_3_replay_both_optimizers": true,
    "test_answers_rescored": 3000,
    "all_rescored_rewards_matched": true,
    "spectral_drift_matrices_checked": 3,
    "max_relative_spectral_drift_rounded": 1.3e-08
  },
  "cost": {
    "approximate_h200_compute_usd": 8.32,
    "scope": "Includes earlier H200 qualification attempts; excludes storage and network charges. Estimate, not an invoice.",
    "approved_total_cloud_ceiling_usd": 50,
    "main_campaign_duration_hours_rounded": 2.87,
    "experiment_vm_and_boot_disk_deleted": true
  },
  "limitations": [
    "One model, one task, one training seed and a short 50-step horizon; not a paper reproduction.",
    "Small learning-rate grid; 12-step selection and 50-step pilots use different linear-decay horizons.",
    "Question bootstrap does not measure training-seed variability.",
    "Held out from this fine-tuning experiment; pretraining contamination was not assessed.",
    "Independent optimizer implementation. The numerical backend benchmark measures the polar component, not full training versus AdamW."
  ]
}