Creating multi language support using registry, analyzer and provider

[magedhelmy1]

Just need a second opinion if I am using the registry, analyzer and provider or I have created spaghetti code?
I want to anonymize both Norwegian and English sentences.
Any improvement is highly appreciated.

Here is my code:

from presidio_anonymizer import AnonymizerEngine
from presidio_analyzer.nlp_engine import NlpEngineProvider
from presidio_analyzer.recognizer_registry import RecognizerRegistry
from presidio_analyzer import AnalyzerEngine
from typing import Tuple
from detect_language import LanguageDetection

SUPPORTED_LANG = ["no", "en"]


class PiiAnonymizer:
    @staticmethod
    def _nlp_config():
        return {
            "nlp_engine_name": "spacy",
            "models": [
                {"lang_code": "en", "model_name": "en_core_web_lg"},
                {"lang_code": "no", "model_name": "nb_core_news_lg"},
            ],
        }

    @staticmethod
    def anonymize(text: str) -> Tuple[str, str, str, int]:
        lang_detect = LanguageDetection()
        detected_lang = lang_detect.detect_lang(text)

        provider = NlpEngineProvider(nlp_configuration=PiiAnonymizer._nlp_config())
        nlp_engine_with_norwegian = provider.create_engine()

        registry = RecognizerRegistry()
        registry.load_predefined_recognizers(
            nlp_engine=nlp_engine_with_norwegian, languages=SUPPORTED_LANG
        )

        analyzer = AnalyzerEngine(
            registry=registry,
            nlp_engine=nlp_engine_with_norwegian,
            supported_languages=SUPPORTED_LANG,
        )

        analyzer_results = analyzer.analyze(text=text, language=detected_lang)

        anonymizer = AnonymizerEngine()
        anonymized_text = anonymizer.anonymize(text, analyzer_results)

        return text, anonymized_text.text, detected_lang


if __name__ == "__main__":
    text = " My name is John Smith, my email address is [email protected], and my phone number is (555) 555-5555 \
        I live in New York. I also have some  cryptocurrency: \
        Bitcoin, Ethereum, and Litecoin. My IBAN code is GB33BUKB20201555555555, and my IP address is 192.168.0.1. \
        I have a Norwegian national registry number (NRP) of 01010112345."

    text2 = "Jeg vil betale deg 500 kr via kredittkortnummer GB33BUKB20201555555555 i morgen i USA klokka 10. Sees da!"
    text, results, lang = PiiAnonymizer.anonymize(text2)

    print(f"Language detected: {lang}")
    print("----------------------------------")
    print(f"Original text: {text}")
    print("----------------------------------")
    print(f"Anonymized text: {results}")
    print("----------------------------------")

And here is the language code if someone is interested:

from spacy.language import Language
from spacy_langdetect import LanguageDetector
import spacy


class LanguageDetection:
    def __init__(self):
        self._nlp = self.load_nlp()

    @Language.factory("language_detector")
    def get_lang_detector(nlp, name):
        return LanguageDetector()

    def detect_lang(self, text):
        self._language = self._nlp(text)._.language["language"]
        return self._language

    @staticmethod
    def load_nlp() -> Language:
        nlp = spacy.load("en_core_web_lg")
        nlp.add_pipe("language_detector", last=True)
        return nlp


if __name__ == "__main__":
    lang_detect = LanguageDetection()
    detected_lang = lang_detect.detect_lang("How are you")
    print(detected_lang)

[omri374]

Hi, looks good to me in terms of the setup, however it is probably better to load the AnalyzerEngine only once as this code would instantiate the spacy models on every call, which is likely going to be slow.

Thanks for sharing your code for language detection! this is very useful.

[magedhelmy1]

Thank you omri374! I appreciate your answer.