fix: adding missing abbreviations files for SentenceSplitter (#8660)

* adding missing abbreviations files for SentenceSplitter * fixing tests path
deepset-ai · Dec 19, 2024 · c306bee · c306bee
1 parent 91619a7
commit c306bee
Show file tree

Hide file tree

Showing 5 changed files with 2,075 additions and 2 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -26,6 +26,7 @@ repos:
     rev: v2.3.0
     hooks:
       - id: codespell
+        exclude: "haystack/data/abbreviations"
         args: ["--toml", "pyproject.toml"]
         additional_dependencies:
           - tomli

diff --git a/haystack/components/preprocessors/sentence_tokenizer.py b/haystack/components/preprocessors/sentence_tokenizer.py
@@ -228,7 +228,7 @@ def _read_abbreviations(lang: Language) -> List[str]:
         :param lang: The language to read the abbreviations for.
         :returns: List of abbreviations.
         """
-        abbreviations_file = Path(__file__).parent.parent / f"data/abbreviations/{lang}.txt"
+        abbreviations_file = Path(__file__).parent.parent.parent / f"data/abbreviations/{lang}.txt"
         if not abbreviations_file.exists():
             logger.warning("No abbreviations file found for {language}. Using default abbreviations.", language=lang)
             return []