Design a Complete Multimodal RLVR Pipeline with Open-MM-RL, Vision-Language Prompting, Reward Scoring, and GRPO Export

import torch
USE_VLM = torch.cuda.is_available()
print(f”CUDA available: {USE_VLM}”)
if USE_VLM:
try:
from transformers import AutoProcessor, AutoModelForVision2Seq
MODEL_ID = “HuggingFaceTB/SmolVLM-Instruct”
print(f”Loading {MODEL_ID} (this takes ~1 min) …”)
processor = AutoProcessor.from_pretrained(MODEL_ID)
model = AutoModelForVision2Seq.from_pretrained(
MODEL_ID, torch_dtype=torch.float16, device_map=”auto”
)
def vlm_solve(ex, max_new_tokens=512):
imgs = [im.convert(“RGB”) for im in ex[“images”]]
content = [{“type”: “image”} for _ in imgs]
content.append({“type”: “text”, “text”: build_prompt(ex)})
text = processor.apply_chat_template(
[{“role”: “user”, “content”: content}], add_generation_prompt=True)
inputs = processor(text=text, images=imgs, return_tensors=”pt”).to(model.device)
with torch.no_grad():
out = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False)
return processor.batch_decode(
out[:, inputs.input_ids.shape[1]:], skip_special_tokens=True)[0]
rows, sample_idx = [], random.sample(range(len(ds)), 6)
for i in sample_idx:
ex = ds[i]
try:
pred = vlm_solve(ex)
r = grade(pred, ex[“answer”])
except Exception as e:
pred, r = f””, 0.0
rows.append({“id”: ex[“conversation_id”], “domain”: ex[“domain”],
“reward”: r, “pred_tail”: pred[-200:]})
print(f” id={ex[‘conversation_id’]} {ex[‘domain’]:9s} r={r:.2f}”)
res = pd.DataFrame(rows)
print(f”\nMean reward over {len(res)} samples: {res[‘reward’].mean():.3f}”)
print(res.groupby(“domain”)[“reward”].mean().rename(“avg_reward”))
except Exception as e:
print(f”VLM run failed ({e}); reward & data pipeline remain usable.”)
else:
print(“No GPU detected — skipping live VLM inference (Runtime → Change runtime type → GPU).”)
out_dir = Path(“/content/open_mm_rl_processed”); out_dir.mkdir(exist_ok=True, parents=True)
img_dir = out_dir / “images”; img_dir.mkdir(exist_ok=True)
records = []
for ex in ds:
paths = []
for j, im in enumerate(ex[“images”]):
p = img_dir / f”{ex[‘conversation_id’]}_{j}.png”
im.convert(“RGB”).save(p)
paths.append(str(p))
records.append({
“id”: ex[“conversation_id”],
“domain”: ex[“domain”],
“subDomain”: ex[“subDomain”],
“format”: ex[“format”],
“prompt”: build_prompt(ex),
“gold”: ex[“answer”],
“image_paths”: paths,
})
jsonl_path = out_dir / “data.jsonl”
with open(jsonl_path, “w”) as f:
for r in records: f.write(json.dumps(r) + “\n”)
print(f”\nWrote {len(records)} records → {jsonl_path}”)
print(f”Saved {sum(len(r[‘image_paths’]) for r in records)} images under {img_dir}”)
def mock_policy_samples(gold, K=4):
“””Stand-in for K policy rollouts. Replace with model.generate(do_sample=True).”””
return [gold,
“Final answer: 0″,
f”Final answer: {gold} (≈)”,
“I think the answer is unclear.”][:K]
def grpo_advantages(rewards):
r = np.asarray(rewards, dtype=float)
return (r – r.mean()) / (r.std() + 1e-6)
print(“\n=== Mock GRPO rollouts for example 0 ===”)
gold0 = ds[0][“answer”]
cands = mock_policy_samples(gold0, K=4)
rewards = [grade(c, gold0) for c in cands]
adv = grpo_advantages(rewards)
for c, r, a in zip(cands, rewards, adv):
print(f” r={r:.2f} adv={a:+.2f} cand={c!r}”)
print(“\nDone. To turn this into real training:”)
print(” 1. Replace mock_policy_samples with vlm_solve(…, do_sample=True, num_return_sequences=K).”)
print(” 2. Feed (prompt, K rollouts, K rewards) into TRL’s GRPOTrainer or verl.”)
print(” 3. Curriculum: start with examples where rewards have non-zero variance.”)