Merge pull request #29 from weni-ai/feature/links

Feature/links
weni-ai · Mar 28, 2024 · b6069eb · b6069eb
2 parents ab663ef + 255f922
commit b6069eb
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 9 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -7,7 +7,7 @@ RUN pip install poetry
 COPY pyproject.toml poetry.lock ./
 
 RUN poetry config virtualenvs.create false && \
-    poetry install --no-dev
+    poetry install --only main
 
 RUN apt update && apt install libmagic1 -y
 RUN poetry add python-magic

diff --git a/app/loaders/loaders.py b/app/loaders/loaders.py
@@ -220,7 +220,7 @@ def _urls(self, urls: Union[List[str], str]):
 
     def __init__(self, urls: Union[List[str], str]) -> None:
         self.urls = self._urls(urls)
-        self.loader = UnstructuredURLLoader(urls=self.urls)
+        self.loader = UnstructuredURLLoader(urls=self.urls, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/109.0"})
 
     def load(self) -> List[Document]:
         return self.loader.load()

diff --git a/poetry.lock b/poetry.lock