Improve cache (#90)

* WIP: warming cache by source and last update * Loading github data based on the rate limit * Adding cached_at logic * Modify weird function spec * Back cache_interval to 1h * Improve the usage of the Backends behaviour
fiqus · Jul 29, 2021 · 0344de9 · 0344de9
1 parent 57a4c06
commit 0344de9
Show file tree

Hide file tree

Showing 11 changed files with 1,018 additions and 7,088 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.2.10
+0.3.0
diff --git a/config/config.exs b/config/config.exs
@@ -20,7 +20,7 @@ config :coophub,
   fetch_max_repos: 10,
   # Configures Cachex
   main_cache_name: :repos_cache,
-  cache_interval: 60,
+  cache_interval: 30,
   main_cache_dump_file: "repos-cache.dump",
   uris_cache_name: :uris_cache
 

diff --git a/lib/coophub/backends/backends.ex b/lib/coophub/backends/backends.ex
@@ -13,6 +13,21 @@ defmodule Coophub.Backends do
   @type langs :: %{String.t() => integer()}
   @type topics :: [String.t()]
 
+  @callback name() :: String.t()
+
+  @callback prepare_request_org(String.t()) :: request_data
+  @callback parse_org(map) :: org
+
+  @callback prepare_request_repos(org, integer) :: request_data
+  @callback prepare_request_repo(org, map) :: request_data
+  @callback parse_repo(map) :: repo
+
+  @callback prepare_request_topics(org, repo) :: request_data
+  @callback parse_topics(any) :: topics
+
+  @callback prepare_request_languages(org, repo) :: request_data
+  @callback parse_languages(any) :: langs
+
   ## Backends implementations
   defp get_backend_module!("github"), do: Backends.Github
   defp get_backend_module!("gitlab"), do: Backends.GitlabCom
@@ -131,6 +146,21 @@ defmodule Coophub.Backends do
     end
   end
 
+  @spec get_rate_limit(String.t()) :: integer
+  def get_rate_limit(source) do
+    backend = get_backend_module!(source)
+    {_, url, headers} = backend.prepare_request_rate_limit()
+
+    case request(url, headers) do
+      {:ok, data, _} ->
+        data["resources"]["core"]["remaining"]
+
+      {:error, reason} ->
+        Logger.error("Error getting account rate limit: #{inspect(reason)}")
+        0
+    end
+  end
+
   @spec request(String.t(), headers) :: {:ok, map | [map], integer} | {:error, any}
   defp request(nil, _), do: {:ok, [], 0}
 

diff --git a/lib/coophub/backends/behaviour.ex b/lib/coophub/backends/behaviour.ex
diff --git a/lib/coophub/backends/github.ex b/lib/coophub/backends/github.ex
@@ -11,29 +11,29 @@ defmodule Coophub.Backends.Github do
   @type langs :: Backends.langs()
   @type topics :: Backends.topics()
 
-  @behaviour Backends.Behaviour
+  @behaviour Backends
 
   ########
   ## BEHAVIOUR IMPLEMENTATION
   ########
 
-  @impl Backends.Behaviour
+  @impl Backends
   @spec name() :: String.t()
   def name(), do: "github"
 
-  @impl Backends.Behaviour
+  @impl Backends
   @spec prepare_request_org(String.t()) :: request_data
   def prepare_request_org(login) do
     prepare_request(login, "orgs/#{login}")
   end
 
-  @impl Backends.Behaviour
+  @impl Backends
   @spec parse_org(map) :: org
   def parse_org(data) do
     Repos.to_struct(Organization, data)
   end
 
-  @impl Backends.Behaviour
+  @impl Backends
   @spec prepare_request_repos(org, integer) :: request_data
   def prepare_request_repos(%Organization{login: login}, limit) do
     prepare_request(
@@ -42,13 +42,13 @@ defmodule Coophub.Backends.Github do
     )
   end
 
-  @impl Backends.Behaviour
+  @impl Backends
   @spec prepare_request_repo(org, map) :: request_data
   def prepare_request_repo(%Organization{login: login}, %{"name" => name}) do
     prepare_request("#{login}/#{name}", "repos/#{login}/#{name}")
   end
 
-  @impl Backends.Behaviour
+  @impl Backends
   @spec parse_repo(map) :: repo
   def parse_repo(data) do
     repo = Repos.to_struct(Repository, data)
@@ -62,30 +62,34 @@ defmodule Coophub.Backends.Github do
     end
   end
 
-  @impl Backends.Behaviour
+  @impl Backends
   @spec prepare_request_topics(org, repo) :: request_data
   def prepare_request_topics(%Organization{login: login}, %Repository{name: name}) do
     prepare_request("#{login}/#{name}", "repos/#{login}/#{name}/topics")
   end
 
-  @impl Backends.Behaviour
+  @impl Backends
   @spec parse_topics(map) :: topics
   def parse_topics(data) do
     Map.get(data, "names", [])
   end
 
-  @impl Backends.Behaviour
+  @impl Backends
   @spec prepare_request_languages(org, repo) :: request_data
   def prepare_request_languages(%Organization{login: login}, %Repository{name: name}) do
     prepare_request("#{login}/#{name}", "repos/#{login}/#{name}/languages")
   end
 
-  @impl Backends.Behaviour
+  @impl Backends
   @spec parse_languages(langs) :: langs
   def parse_languages(languages) do
     percentage_from_bytes(languages)
   end
 
+  def prepare_request_rate_limit() do
+    prepare_request("", "rate_limit")
+  end
+
   ########
   ## INTERNALS
   ########

diff --git a/lib/coophub/backends/gitlab.ex b/lib/coophub/backends/gitlab.ex
@@ -11,15 +11,15 @@ defmodule Coophub.Backends.Gitlab do
       @type langs :: Backends.langs()
       @type topics :: Backends.topics()
 
-      @behaviour Backends.Behaviour
+      @behaviour Backends
 
-      @impl Backends.Behaviour
+      @impl Backends
       @spec prepare_request_org(String.t()) :: request_data
       def prepare_request_org(login) do
         prepare_request(login, "groups/#{login}")
       end
 
-      @impl Backends.Behaviour
+      @impl Backends
       @spec parse_org(map) :: org
       def parse_org(data) do
         data =
@@ -34,7 +34,7 @@ defmodule Coophub.Backends.Gitlab do
         Repos.to_struct(Organization, data)
       end
 
-      @impl Backends.Behaviour
+      @impl Backends
       @spec prepare_request_repos(org, integer) :: request_data
       def prepare_request_repos(%Organization{login: login}, limit) do
         prepare_request(
@@ -43,7 +43,7 @@ defmodule Coophub.Backends.Gitlab do
         )
       end
 
-      @impl Backends.Behaviour
+      @impl Backends
       @spec prepare_request_repo(org, map) :: request_data
       def prepare_request_repo(_organization, %{"path_with_namespace" => path_with_namespace}) do
         prepare_request(
@@ -52,7 +52,7 @@ defmodule Coophub.Backends.Gitlab do
         )
       end
 
-      @impl Backends.Behaviour
+      @impl Backends
       @spec parse_repo(map) :: repo
       def parse_repo(data) do
         data =
@@ -84,20 +84,20 @@ defmodule Coophub.Backends.Gitlab do
         end
       end
 
-      @impl Backends.Behaviour
+      @impl Backends
       @spec prepare_request_topics(org, repo) :: request_data
       def prepare_request_topics(_, _) do
         # topics are tag_list already set
         dont_request()
       end
 
-      @impl Backends.Behaviour
+      @impl Backends
       @spec parse_topics(map) :: topics
       def parse_topics(data) do
         data
       end
 
-      @impl Backends.Behaviour
+      @impl Backends
       @spec prepare_request_languages(org, repo) :: request_data
       def prepare_request_languages(_organization, %Repository{
             path_with_namespace: path_with_namespace
@@ -108,7 +108,7 @@ defmodule Coophub.Backends.Gitlab do
         )
       end
 
-      @impl Backends.Behaviour
+      @impl Backends
       @spec parse_languages(langs) :: langs
       def parse_languages(languages) do
         languages