Create table with Language info based on paper titles

[chrished]

https://fasttext.cc/

Following example from Flavio:

import re
import fasttext


try:
    from importlib.resources import files
except ImportError: # python < 3.10. https://setuptools.pypa.io/en/latest/userguide/datafiles.html
    from importlib_resources import files


fasttext_model_path = str(files('ivory_data_forge.data').joinpath("lid.176.ftz")) # this works only for python >=3.10


print(fasttext_model_path)
model = fasttext.load_model(fasttext_model_path)

def detect_language(s):
    """Detect whether a string is in English, Dutch or another language.

    When used on short strings (such as job titles), the confidence can be quite low (0.2) 
    even though reading the string makes clear that the prediction is correct.  
    
    Args:
        s (str): Text to infer language from.  
    """
    predictions = model.predict(s) 
    language = re.sub("__label__", "", predictions[0][0])
    # confidence = predictions[1][0] 
    if language in ["nl", "en"]:
        return language 
    return "other"