From 4f475a8c9efb2db7f4e0cf9374c4ca3a424dfd8e Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Wed, 25 Jan 2023 14:44:16 -0800 Subject: [PATCH 1/4] deleted storage controller and added hub stats controller --- lib/ret/node_stat.ex | 3 +++ .../api-internal/v1/hub_stats_controller.ex | 24 ++++++++++++++++++ .../api-internal/v1/storage_controller.ex | 15 ----------- lib/ret_web/router.ex | 25 ++++++++++++------- .../api-internal/storage_controller_test.exs | 6 ++++- 5 files changed, 48 insertions(+), 25 deletions(-) create mode 100644 lib/ret_web/controllers/api-internal/v1/hub_stats_controller.ex delete mode 100644 lib/ret_web/controllers/api-internal/v1/storage_controller.ex diff --git a/lib/ret/node_stat.ex b/lib/ret/node_stat.ex index 0bb030493..064f6ceb3 100644 --- a/lib/ret/node_stat.ex +++ b/lib/ret/node_stat.ex @@ -24,6 +24,7 @@ defmodule Ret.NodeStat do ]) end + @spec max_ccu_for_time_range(DateTime.t(), DateTime.t()) :: number() def max_ccu_for_time_range(start_time, end_time) do start_time_truncated = start_time |> NaiveDateTime.truncate(:second) end_time_truncated = end_time |> NaiveDateTime.truncate(:second) @@ -35,6 +36,8 @@ defmodule Ret.NodeStat do ) |> Repo.one() + # Can I check that the db actually has the time range? So we could technically fill in previous data. + if max_ccu === nil, do: 0, else: max_ccu end end diff --git a/lib/ret_web/controllers/api-internal/v1/hub_stats_controller.ex b/lib/ret_web/controllers/api-internal/v1/hub_stats_controller.ex new file mode 100644 index 000000000..dc9da2904 --- /dev/null +++ b/lib/ret_web/controllers/api-internal/v1/hub_stats_controller.ex @@ -0,0 +1,24 @@ +defmodule RetWeb.ApiInternal.V1.HubStatsController do + use RetWeb, :controller + alias Ret.NodeStat + + # Params start_time and end_time should be in iso format such as "2000-02-28 23:00:13" + # or what is returned from NaiveDateTime.to_string() + def hub_stats(conn, %{"start_time" => start_time_str, "end_time" => end_time_str}) do + conn = put_resp_header(conn, "content-type", "application/json") + + case Ret.Storage.storage_used() do + {:ok, storage_used_kb} when is_number(storage_used_kb) -> + max_ccu = + NodeStat.max_ccu_for_time_range( + start_time_str |> NaiveDateTime.from_iso8601!(), + end_time_str |> NaiveDateTime.from_iso8601!() + ) + + conn |> send_resp(200, %{max_ccu: max_ccu, storage_mb: storage_used_kb / 1024} |> Poison.encode!()) + + _ -> + send_resp(conn, 503, %{error: :storage_usage_unavailable} |> Poison.encode!()) + end + end +end diff --git a/lib/ret_web/controllers/api-internal/v1/storage_controller.ex b/lib/ret_web/controllers/api-internal/v1/storage_controller.ex deleted file mode 100644 index 9cdd3e2c3..000000000 --- a/lib/ret_web/controllers/api-internal/v1/storage_controller.ex +++ /dev/null @@ -1,15 +0,0 @@ -defmodule RetWeb.ApiInternal.V1.StorageController do - use RetWeb, :controller - - def show(conn, _) do - conn = put_resp_header(conn, "content-type", "application/json") - - case Ret.Storage.storage_used() do - {:ok, storage_used_kb} when is_number(storage_used_kb) -> - send_resp(conn, 200, %{storage_mb: storage_used_kb / 1024} |> Poison.encode!()) - - _ -> - send_resp(conn, 503, %{error: :storage_usage_unavailable} |> Poison.encode!()) - end - end -end diff --git a/lib/ret_web/router.ex b/lib/ret_web/router.ex index 16c6954d7..51d2142f0 100644 --- a/lib/ret_web/router.ex +++ b/lib/ret_web/router.ex @@ -83,8 +83,8 @@ defmodule RetWeb.Router do end pipeline :graphql do - plug RetWeb.ApiTokenAuthPipeline - plug RetWeb.AddAbsintheContext + plug(RetWeb.ApiTokenAuthPipeline) + plug(RetWeb.AddAbsintheContext) end scope "/health", RetWeb do @@ -103,7 +103,8 @@ defmodule RetWeb.Router do scope "/api", RetWeb do pipe_through( - [:secure_headers, :parsed_body, :api] ++ if(Mix.env() == :prod, do: [:ssl_only, :canonicalize_domain], else: []) + [:secure_headers, :parsed_body, :api] ++ + if(Mix.env() == :prod, do: [:ssl_only, :canonicalize_domain], else: []) ) scope "/v1", as: :api_v1 do @@ -114,6 +115,7 @@ defmodule RetWeb.Router do scope "/support" do resources("/subscriptions", Api.V1.SupportSubscriptionController, only: [:create, :delete]) + resources("/availability", Api.V1.SupportSubscriptionController, only: [:index]) end @@ -185,23 +187,25 @@ defmodule RetWeb.Router do scope "/api/v2_alpha", as: :api_v2_alpha do pipe_through( - [:parsed_body, :api, :public_api_access, :graphql] ++ if(Mix.env() == :prod, do: [:ssl_only], else: []) + [:parsed_body, :api, :public_api_access, :graphql] ++ + if(Mix.env() == :prod, do: [:ssl_only], else: []) ) - forward "/graphiql", Absinthe.Plug.GraphiQL, json_codec: Jason, schema: RetWeb.Schema - forward "/", Absinthe.Plug, json_codec: Jason, schema: RetWeb.Schema + forward("/graphiql", Absinthe.Plug.GraphiQL, json_codec: Jason, schema: RetWeb.Schema) + forward("/", Absinthe.Plug, json_codec: Jason, schema: RetWeb.Schema) end scope "/api-internal", RetWeb do pipe_through( - [:dashboard_header_auth, :secure_headers, :parsed_body, :api] ++ if(Mix.env() == :prod, do: [:ssl_only], else: []) + [:dashboard_header_auth, :secure_headers, :parsed_body, :api] ++ + if(Mix.env() == :prod, do: [:ssl_only], else: []) ) scope "/v1", as: :api_internal_v1 do get("/presence", ApiInternal.V1.PresenceController, :show) get("/presence/range_max", ApiInternal.V1.PresenceController, :range_max) - get("/storage", ApiInternal.V1.StorageController, :show) post("/rewrite_assets", ApiInternal.V1.RewriteAssetsController, :post) + get("/hub_stats", ApiInternal.V1.HubStatsController, :show) end end @@ -216,7 +220,10 @@ defmodule RetWeb.Router do end scope "/", RetWeb do - pipe_through([:strict_secure_headers, :parsed_body, :browser] ++ if(Mix.env() == :prod, do: [:ssl_only], else: [])) + pipe_through( + [:strict_secure_headers, :parsed_body, :browser] ++ + if(Mix.env() == :prod, do: [:ssl_only], else: []) + ) head("/files/:id", FileController, :head) get("/files/:id", FileController, :show) diff --git a/test/ret_web/controllers/api-internal/storage_controller_test.exs b/test/ret_web/controllers/api-internal/storage_controller_test.exs index b6ba7f987..3cc12decc 100644 --- a/test/ret_web/controllers/api-internal/storage_controller_test.exs +++ b/test/ret_web/controllers/api-internal/storage_controller_test.exs @@ -1,4 +1,4 @@ -defmodule RetWeb.ApiInternal.V1.StorageControllerTest do +defmodule RetWeb.ApiInternal.V1.HubStatsControllerTest do use RetWeb.ConnCase import Ret.TestHelpers @@ -47,6 +47,10 @@ defmodule RetWeb.ApiInternal.V1.StorageControllerTest do defp mock_storage_used(nil), do: Cachex.put(:storage_used, :storage_used, nil) defp mock_storage_used(storage_used_mb), do: Cachex.put(:storage_used, :storage_used, storage_used_mb * 1024) + defp seed_ccu() do + # todo + end + defp request_storage(conn, opts \\ [expected_status: 200]) do conn |> put_req_header(@dashboard_access_header, @dashboard_access_key) From ddad35122938edb2795366bd8e68d90a36eefa83 Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Wed, 25 Jan 2023 15:10:49 -0800 Subject: [PATCH 2/4] rename wrong standard named file --- .../v1/{AuthTokenController.ex => auth_token_controller.ex} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/ret_web/controllers/api-internal/v1/{AuthTokenController.ex => auth_token_controller.ex} (100%) diff --git a/lib/ret_web/controllers/api-internal/v1/AuthTokenController.ex b/lib/ret_web/controllers/api-internal/v1/auth_token_controller.ex similarity index 100% rename from lib/ret_web/controllers/api-internal/v1/AuthTokenController.ex rename to lib/ret_web/controllers/api-internal/v1/auth_token_controller.ex From 8585dff56300bd781f6dfe0be397fd12ceeaa33f Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Mon, 6 Feb 2023 13:59:31 -0800 Subject: [PATCH 3/4] added old storage endpoint and max ccu back in --- .../api-internal/v1/presence_controller.ex | 12 ++++ .../api-internal/v1/storage_controller.ex | 15 ++++ lib/ret_web/router.ex | 2 + .../hub_stats_controller_test.exs | 68 +++++++++++++++++++ .../api-internal/storage_controller_test.exs | 34 +++++++--- 5 files changed, 120 insertions(+), 11 deletions(-) create mode 100644 lib/ret_web/controllers/api-internal/v1/storage_controller.ex create mode 100644 test/ret_web/controllers/api-internal/hub_stats_controller_test.exs diff --git a/lib/ret_web/controllers/api-internal/v1/presence_controller.ex b/lib/ret_web/controllers/api-internal/v1/presence_controller.ex index 68bb27940..66e1c031e 100644 --- a/lib/ret_web/controllers/api-internal/v1/presence_controller.ex +++ b/lib/ret_web/controllers/api-internal/v1/presence_controller.ex @@ -8,4 +8,16 @@ defmodule RetWeb.ApiInternal.V1.PresenceController do conn |> send_resp(200, %{count: count} |> Poison.encode!()) end + + # Params start_time and end_time should be in iso format such as "2000-02-28 23:00:13" + # or what is returned from NaiveDateTime.to_string() + def range_max(conn, %{"start_time" => start_time_str, "end_time" => end_time_str}) do + max = + NodeStat.max_ccu_for_time_range( + start_time_str |> NaiveDateTime.from_iso8601!(), + end_time_str |> NaiveDateTime.from_iso8601!() + ) + + conn |> send_resp(200, %{max_ccu: max} |> Poison.encode!()) + end end diff --git a/lib/ret_web/controllers/api-internal/v1/storage_controller.ex b/lib/ret_web/controllers/api-internal/v1/storage_controller.ex new file mode 100644 index 000000000..9cdd3e2c3 --- /dev/null +++ b/lib/ret_web/controllers/api-internal/v1/storage_controller.ex @@ -0,0 +1,15 @@ +defmodule RetWeb.ApiInternal.V1.StorageController do + use RetWeb, :controller + + def show(conn, _) do + conn = put_resp_header(conn, "content-type", "application/json") + + case Ret.Storage.storage_used() do + {:ok, storage_used_kb} when is_number(storage_used_kb) -> + send_resp(conn, 200, %{storage_mb: storage_used_kb / 1024} |> Poison.encode!()) + + _ -> + send_resp(conn, 503, %{error: :storage_usage_unavailable} |> Poison.encode!()) + end + end +end diff --git a/lib/ret_web/router.ex b/lib/ret_web/router.ex index 812f372d6..3a9ae07bc 100644 --- a/lib/ret_web/router.ex +++ b/lib/ret_web/router.ex @@ -203,6 +203,8 @@ defmodule RetWeb.Router do scope "/v1", as: :api_internal_v1 do get "/presence", ApiInternal.V1.PresenceController, :show + get "/presence/range_max", ApiInternal.V1.PresenceController, :range_max + get "/storage", ApiInternal.V1.StorageController, :show post "/rewrite_assets", ApiInternal.V1.RewriteAssetsController, :post put "/change_email_for_login", ApiInternal.V1.LoginEmailController, :update post "/make_auth_token_for_email", ApiInternal.V1.AuthTokenController, :post diff --git a/test/ret_web/controllers/api-internal/hub_stats_controller_test.exs b/test/ret_web/controllers/api-internal/hub_stats_controller_test.exs new file mode 100644 index 000000000..45425609c --- /dev/null +++ b/test/ret_web/controllers/api-internal/hub_stats_controller_test.exs @@ -0,0 +1,68 @@ +defmodule RetWeb.ApiInternal.V1.StorageControllerTest do + use RetWeb.ConnCase + import Ret.TestHelpers + + @dashboard_access_header "x-ret-dashboard-access-key" + @dashboard_access_key "test-key" + + setup_all do + merge_module_config(:ret, RetWeb.Plugs.DashboardHeaderAuthorization, %{ + dashboard_access_key: @dashboard_access_key + }) + + on_exit(fn -> + Ret.TestHelpers.merge_module_config(:ret, RetWeb.Plugs.DashboardHeaderAuthorization, %{ + dashboard_access_key: nil + }) + end) + end + + test "hub stats endpoint responds with cached storage value", %{conn: conn} do + mock_storage_used(0) + resp = request_hub_stats(conn) + assert resp["storage_mb"] === 0.0 and resp["max_ccu"] === 0.0 + + mock_storage_used(10) + resp = request_hub_stats(conn) + assert resp["storage_mb"] === 10.0 and resp["max_ccu"] === 0.0 + end + + + test "hub stats endpoint returns 401 without access key header", %{conn: conn} do + resp = get(conn, "/api-internal/v1/hub_stats") + assert resp.status === 401 + end + + test "hub stats endpoint returns 401 with incorrect access key", %{conn: conn} do + resp = + conn + |> put_req_header(@dashboard_access_header, "incorrect-access-key") + |> get("/api-internal/v1/hub_stats") + + assert resp.status === 401 + end + + test "hub stats endpoint errors with 503 status when storage usage is not available", %{conn: conn} do + mock_storage_used(nil) + resp = request_hub_stats(conn, expected_status: 503) + assert resp["error"] === "storage_usage_unavailable" + end + + # The Ret.Storage module relies on a cached value to retrieve storage usage via Ret.StorageUsed. + # Since we mainly care about testing the endpoint here, we use the cache to mock the usage value + # and ensure that the endpoint returns it as expected. + defp mock_storage_used(nil), do: Cachex.put(:storage_used, :storage_used, nil) + + defp mock_storage_used(storage_used_mb), + do: Cachex.put(:storage_used, :storage_used, storage_used_mb * 1024) + + defp request_hub_stats(conn, opts \\ [expected_status: 200]) do + {:ok, start_time} = NaiveDateTime.utc_now() |> NaiveDateTime.to_date() |> NaiveDateTime.new(Time.new(0,0,0,0)) + {:ok, end_time} = NaiveDateTime.utc_now() |> NaiveDateTime.to_date() |> Date.add(1) |> NaiveDateTime.new(Time.new(0,0,0,0)) + + conn + |> put_req_header(@dashboard_access_header, @dashboard_access_key) + |> get("/api-internal/v1/hub_stats", %{start_time: start_time, end_time: end_time}) + |> json_response(opts[:expected_status]) + end +end diff --git a/test/ret_web/controllers/api-internal/storage_controller_test.exs b/test/ret_web/controllers/api-internal/storage_controller_test.exs index 57f8c8c6e..8498eb126 100644 --- a/test/ret_web/controllers/api-internal/storage_controller_test.exs +++ b/test/ret_web/controllers/api-internal/storage_controller_test.exs @@ -1,4 +1,4 @@ -defmodule RetWeb.ApiInternal.V1.HubStatsControllerTest do +defmodule RetWeb.ApiInternal.V1.StorageControllerTest do use RetWeb.ConnCase import Ret.TestHelpers @@ -17,34 +17,46 @@ defmodule RetWeb.ApiInternal.V1.HubStatsControllerTest do end) end - test "storage endpoint responds with cached storage value", %{conn: conn} do + test "hub stats endpoint responds with cached storage value", %{conn: conn} do mock_storage_used(0) - resp = request_storage(conn) + resp = request_hub_stats(conn) assert resp["storage_mb"] === 0.0 mock_storage_used(10) - resp = request_storage(conn) + resp = request_hub_stats(conn) assert resp["storage_mb"] === 10.0 end - test "storage endpoint errors without correct access key", %{conn: conn} do - resp = get(conn, "/api-internal/v1/storage") + + test "hub stats endpoint returns 401 without access key header", %{conn: conn} do + resp = get(conn, "/api-internal/v1/hub_stats") assert resp.status === 401 + end + test "hub stats endpoint returns 401 with incorrect access key", %{conn: conn} do resp = conn |> put_req_header(@dashboard_access_header, "incorrect-access-key") - |> get("/api-internal/v1/storage") + |> get("/api-internal/v1/hub_stats") assert resp.status === 401 end - test "storage endpoint errors when storage usage is not available", %{conn: conn} do + test "hub stats endpoint errors with 503 status when storage usage is not available", %{conn: conn} do mock_storage_used(nil) - resp = request_storage(conn, expected_status: 503) + resp = request_hub_stats(conn, expected_status: 503) assert resp["error"] === "storage_usage_unavailable" end + test "hub stats endpoint returns correct ccu numbers for time range", %{conn: conn} do + end + + test "hub stats endpoint returns 0 ccu numbers for time range if no data available", %{conn: conn} do + end + + test "hub stats endpoint returns correct ccu numbers for time range when there is multiple ccu numbers in db", %{conn: conn} do + end + # The Ret.Storage module relies on a cached value to retrieve storage usage via Ret.StorageUsed. # Since we mainly care about testing the endpoint here, we use the cache to mock the usage value # and ensure that the endpoint returns it as expected. @@ -57,10 +69,10 @@ defmodule RetWeb.ApiInternal.V1.HubStatsControllerTest do # todo end - defp request_storage(conn, opts \\ [expected_status: 200]) do + defp request_hub_stats(conn, opts \\ [expected_status: 200]) do conn |> put_req_header(@dashboard_access_header, @dashboard_access_key) - |> get("/api-internal/v1/storage") + |> get("/api-internal/v1/hub_stats?") |> json_response(opts[:expected_status]) end end From 83d1b4fb8ce9fa8ea37bfb16753c6f83d39ccb2b Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Mon, 6 Feb 2023 14:02:25 -0800 Subject: [PATCH 4/4] reverted storage tests --- .../api-internal/storage_controller_test.exs | 36 ++++++------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/test/ret_web/controllers/api-internal/storage_controller_test.exs b/test/ret_web/controllers/api-internal/storage_controller_test.exs index 8498eb126..d027b07a0 100644 --- a/test/ret_web/controllers/api-internal/storage_controller_test.exs +++ b/test/ret_web/controllers/api-internal/storage_controller_test.exs @@ -17,46 +17,34 @@ defmodule RetWeb.ApiInternal.V1.StorageControllerTest do end) end - test "hub stats endpoint responds with cached storage value", %{conn: conn} do + test "storage endpoint responds with cached storage value", %{conn: conn} do mock_storage_used(0) - resp = request_hub_stats(conn) + resp = request_storage(conn) assert resp["storage_mb"] === 0.0 mock_storage_used(10) - resp = request_hub_stats(conn) + resp = request_storage(conn) assert resp["storage_mb"] === 10.0 end - - test "hub stats endpoint returns 401 without access key header", %{conn: conn} do - resp = get(conn, "/api-internal/v1/hub_stats") + test "storage endpoint errors without correct access key", %{conn: conn} do + resp = get(conn, "/api-internal/v1/storage") assert resp.status === 401 - end - test "hub stats endpoint returns 401 with incorrect access key", %{conn: conn} do resp = conn |> put_req_header(@dashboard_access_header, "incorrect-access-key") - |> get("/api-internal/v1/hub_stats") + |> get("/api-internal/v1/storage") assert resp.status === 401 end - test "hub stats endpoint errors with 503 status when storage usage is not available", %{conn: conn} do + test "storage endpoint errors when storage usage is not available", %{conn: conn} do mock_storage_used(nil) - resp = request_hub_stats(conn, expected_status: 503) + resp = request_storage(conn, expected_status: 503) assert resp["error"] === "storage_usage_unavailable" end - test "hub stats endpoint returns correct ccu numbers for time range", %{conn: conn} do - end - - test "hub stats endpoint returns 0 ccu numbers for time range if no data available", %{conn: conn} do - end - - test "hub stats endpoint returns correct ccu numbers for time range when there is multiple ccu numbers in db", %{conn: conn} do - end - # The Ret.Storage module relies on a cached value to retrieve storage usage via Ret.StorageUsed. # Since we mainly care about testing the endpoint here, we use the cache to mock the usage value # and ensure that the endpoint returns it as expected. @@ -65,14 +53,10 @@ defmodule RetWeb.ApiInternal.V1.StorageControllerTest do defp mock_storage_used(storage_used_mb), do: Cachex.put(:storage_used, :storage_used, storage_used_mb * 1024) - defp seed_ccu() do - # todo - end - - defp request_hub_stats(conn, opts \\ [expected_status: 200]) do + defp request_storage(conn, opts \\ [expected_status: 200]) do conn |> put_req_header(@dashboard_access_header, @dashboard_access_key) - |> get("/api-internal/v1/hub_stats?") + |> get("/api-internal/v1/storage") |> json_response(opts[:expected_status]) end end