[QUESTION] get different reward scores for the same prompt

[Cppowboy]

When I use the script below to obtain reward scores, the reward model provides varying scores for the same prompt.

from transformers import AutoTokenizer
import torch
from pytriton.client import ModelClient
import numpy as np
from tqdm import tqdm
from itertools import islice
from loguru import logger
import os
import json


def infer_batch(client, texts, tokenizer):
    input_ids = tokenizer(texts)["input_ids"]
    lengths = [len(x) for x in input_ids]
    max_len = max(lengths)
    padded_input_ids = [x + [0] * (max_len - length) for x, length in zip(input_ids, lengths)]
    tokens = np.array(padded_input_ids, dtype=np.int64)
    sequence_lengths = np.array([[x] for x in lengths], dtype=np.int64)
    result = client.infer_batch(tokens=tokens, sequence_lengths=sequence_lengths)
    return result["rewards"]


def read_jsonl(file):
    return [json.loads(line) for line in open(file)]


def write_jsonl(data, file):
    with open(file, "w") as f:
        for x in data:
            print(json.dumps(x, ensure_ascii=False), file=f)


tokenizer = AutoTokenizer.from_pretrained("Qwen2.5-7B-Instruct")

# 创建ModelClient实例
client = ModelClient(url="localhost:5555", model_name="reward_model")
messages = [{"role": "user", "content": "hello"}, {"role": "assistant", "content": "How are you?"}]
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)

rewards = infer_batch(client, [prompt, prompt, prompt], tokenizer)
print(rewards)

# 关闭客户端连接
client.close()
logger.info("all done")

And the result is

[[-1.7553203]
 [-1.5532155]
 [-1.6571374]]