fix: news conflicts

Gowtham1729 · Dec 10, 2023 · 4feb8b2 · 4feb8b2
1 parent 63f5043
commit 4feb8b2
Showing 1 changed file with 19 additions and 8 deletions.
diff --git a/applications/data_fetcher/fetcher.py b/applications/data_fetcher/fetcher.py
@@ -45,9 +45,7 @@ def __init__(self):
             user=DB_USER,
             password=DB_PASSWORD,
         )
-        self.marketaux_news_url = (
-            f"https://api.marketaux.com/v1/news/all?api_token={MARKETAUX_API_KEY}"
-        )
+        self.marketaux_news_url = f"https://api.marketaux.com/v1/news/all"
         self.cursor = self.connection.cursor()
 
     def get_tickers(self) -> List[Ticker]:
@@ -63,6 +61,7 @@ def insert_news(self, news: List[News]):
                         news_news 
                         (category, symbol, src, src_url, img_src_url, headline, summary, publish_time, sentiment) 
                         VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+                        ON CONFLICT DO NOTHING
                         """,
                 (
                     item.category,
@@ -108,28 +107,40 @@ def fetch_news(self):
         ticker_symbols = ",".join(tickers_list)
         today_date = datetime.today().strftime("%Y-%m-%d")
         page = 1
-        news_url = f"{self.marketaux_news_url}&symbols={ticker_symbols}&published_on={today_date}"
+        news_url = f"{self.marketaux_news_url}?symbols={ticker_symbols}&published_on={today_date}"
 
-        logger.info(f"Fetching News...")
-        response = requests.get(f"{news_url}&page={page}")
+        logger.info(f"Fetching News from the URl {news_url} ...")
+        response = requests.get(f"{news_url}&page={page}&api_token={MARKETAUX_API_KEY}")
         news = []
         if response.status_code == 200:
             response_json = response.json()
             news += self.to_news(response_json["data"], tickers_list)
+            logger.info(f"Total News: {response_json['meta']['found']}")
 
             while (
                 page < 25
                 and page
                 < response_json["meta"]["found"] // response_json["meta"]["limit"]
             ):
                 page += 1
-                response = requests.get(f"{news_url}&page={page}")
+                logger.info(f"Fetching News from the URl {news_url}&page={page} ...")
+                response = requests.get(
+                    f"{news_url}&page={page}&api_token={MARKETAUX_API_KEY}"
+                )
                 if response.status_code == 200:
                     response_json = response.json()
                     news += self.to_news(response_json["data"], tickers_list)
                 else:
+                    logger.error(
+                        f"Error fetching news from the URL {news_url}&page={page}"
+                    )
+                    logger.error(f"Response: {response.json()}")
                     break
-        logger.info(f"Finished fetching News: {news}")
+        else:
+            logger.error(f"Error fetching news from the URL {news_url}")
+            logger.error(f"Response: {response.json()}")
+
+        logger.info(f"Finished fetching News: {[n.headline for n in news]}")
         logger.info(f"Inserting News...")
         self.insert_news(news)