diff --git a/CybersecurityBenchmarks/benchmark/llm.py b/CybersecurityBenchmarks/benchmark/llm.py
index 27a0a203..0991fa4b 100644
--- a/CybersecurityBenchmarks/benchmark/llm.py
+++ b/CybersecurityBenchmarks/benchmark/llm.py
@@ -381,6 +381,11 @@ def query_multimodal_with_retries(
                         self.model, system_prompt, cache_key, response
                     )
                 return response
+            except NotImplementedError as exception:
+                LOG.error(
+                    f"LLM Query failed as the method was not implemented: {exception}"
+                )
+                raise exception
             except UnretriableQueryException as exception:
                 LOG.error(f"LLM Query failed due to unretriable error: {exception}")
                 return None