Skip to content

Commit

Permalink
Added answer an ideal to gradeable environments (#762)
Browse files Browse the repository at this point in the history
  • Loading branch information
whitead authored Dec 11, 2024
1 parent 5eaa9ee commit a630d92
Showing 1 changed file with 12 additions and 1 deletion.
13 changes: 12 additions & 1 deletion paperqa/agents/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ def __init__(
)
self._evaluation_callback = evaluation_callback
self._rewards = rewards
self.answer = ""
self.ideal = ""

async def validate_sources(
self, manifest_or_index: dict[str, DocDetails] | SearchIndex | None = None
Expand Down Expand Up @@ -141,10 +143,19 @@ async def step(
evaluation = await self._evaluation_from_answer(self.state.session.answer)
if evaluation_callback := self._evaluation_callback:
await evaluation_callback(evaluation)
self.answer = evaluation.answer or ""
self.ideal = evaluation.ideal or ""
return messages, reward + self._rewards[evaluation.value], done, truncated

def export_frame(self) -> Frame:
raise NotImplementedError("Didn't yet need to export a frame.")
return Frame(
state=self.state,
info={
"query": self._query,
"answer": self.answer,
"ideal": self.ideal,
},
)

def __deepcopy__(self, memo) -> Self:
copy_state = deepcopy(self.state, memo)
Expand Down

0 comments on commit a630d92

Please sign in to comment.