diff --git a/nginx-lua-prometheus-0.20240525-1.rockspec b/nginx-lua-prometheus-0.20260121-1.rockspec similarity index 55% rename from nginx-lua-prometheus-0.20240525-1.rockspec rename to nginx-lua-prometheus-0.20260121-1.rockspec index 3aaf07a..cb8231b 100644 --- a/nginx-lua-prometheus-0.20240525-1.rockspec +++ b/nginx-lua-prometheus-0.20260121-1.rockspec @@ -1,21 +1,18 @@ --- Note, this file must have version in its name --- (see https://github.com/knyar/nginx-lua-prometheus/issues/27) package = "nginx-lua-prometheus" -version = "0.20240525-1" +version = "0.20260121-1" source = { - url = "git+https://github.com/knyar/nginx-lua-prometheus.git", - tag = "0.20240525", + url = "git+https://github.com/solidwall/nginx-lua-prometheus.git", } description = { summary = "Prometheus metric library for Nginx", - homepage = "https://github.com/knyar/nginx-lua-prometheus", - license = "MIT", + homepage = "https://github.com/solidwall/nginx-lua-prometheus", } dependencies = { "lua >= 5.1", + "lua-resty-lock", } build = { diff --git a/prometheus.lua b/prometheus.lua index a1848a8..64d6e63 100644 --- a/prometheus.lua +++ b/prometheus.lua @@ -56,6 +56,7 @@ -- increments. Copied from https://github.com/Kong/lua-resty-counter local resty_counter_lib = require("prometheus_resty_counter") local key_index_lib = require("prometheus_keys") +local lock = require "resty.lock" local ngx = ngx local ngx_re_match = ngx.re.match local ngx_re_gsub = ngx.re.gsub @@ -456,7 +457,7 @@ end local function inc_gauge(self, value, label_values) local k, err, _, forcible k, err = lookup_or_create(self, label_values) - if err then + if err and self._key_index.first_synced then self._log_error(err) return end @@ -695,12 +696,14 @@ end -- Args: -- dict_name: (string) name of the nginx shared dictionary which will be -- used to store all metrics +-- lock_dict_name: name of the nginx shared dictionary which will be +-- used to store keys for lua-resty-lock lock object -- prefix: (optional string) if supplied, prefix is added to all -- metric names on output -- -- Returns: -- an object that should be used to register metrics. -function Prometheus.init(dict_name, options_or_prefix) +function Prometheus.init(dict_name, lock_dict_name, options_or_prefix) if ngx.get_phase() ~= 'init' and ngx.get_phase() ~= 'init_worker' then error('Prometheus.init can only be called from ' .. 'init_by_lua_block or init_worker_by_lua_block', 2) @@ -709,6 +712,13 @@ function Prometheus.init(dict_name, options_or_prefix) local self = setmetatable({}, mt) dict_name = dict_name or "prometheus_metrics" self.dict_name = dict_name + self.lock, _ = lock:new(lock_dict_name, { + exptime = 60 * 60, -- hour + timeout = 0, -- non-blocking + }) + if not self.lock then + error("Failed to create lock for synching metrics: ") + end self.dict = ngx.shared[dict_name] if self.dict == nil then error("Dictionary '" .. dict_name .. "' does not seem to exist. " .. @@ -728,7 +738,7 @@ function Prometheus.init(dict_name, options_or_prefix) end self.registry = {} - self.key_index = key_index_lib.new(self.dict, KEY_INDEX_PREFIX, function(metric_key) + self.key_index = key_index_lib.new(self.dict, self.lock, KEY_INDEX_PREFIX, function(metric_key) -- When another worker calls reset or del on a metric, reset that -- metric's local lookup table. local metric_name = ngx_re_gsub(metric_key, "{.*", "", "jo") @@ -750,7 +760,7 @@ function Prometheus.init(dict_name, options_or_prefix) self:counter(self.error_metric_name, "Number of nginx-lua-prometheus errors") self.dict:set(self.error_metric_name, 0) local err = self.key_index:add(self.error_metric_name, ERR_MSG_LRU_EVICTION) - if err then + if err and self.key_index.first_synced then self:log_error(err) end @@ -788,9 +798,16 @@ function Prometheus:init_worker(sync_interval) end self._counter = counter_instance - ngx.timer.every(self.sync_interval, function (_) - self.key_index:sync() - end) + local resync + resync = function(_) + self.key_index:sync() + if not self.key_index.first_synced then + ngx.timer.at(0.01, resync) + else + ngx.timer.at(self.sync_interval, resync) + end + end + ngx.timer.at(0.01, resync) end -- Register a new metric. @@ -924,6 +941,9 @@ function Prometheus:metric_data() ngx.log(ngx.ERR, "Prometheus module has not been initialized") return end + if not self.key_index.first_synced then + return + end -- Force a manual sync of counter local state (mostly to make tests work). self._counter:sync() @@ -972,6 +992,9 @@ end -- It will get the metrics from the dictionary, sort them, and expose them -- aling with TYPE and HELP comments. function Prometheus:collect() + if not self.key_index.first_synced then + ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE) -- 503 status code + end ngx.header.content_type = "text/plain" ngx.print(self:metric_data()) end diff --git a/prometheus_keys.lua b/prometheus_keys.lua index 9013652..8d4748a 100644 --- a/prometheus_keys.lua +++ b/prometheus_keys.lua @@ -8,9 +8,10 @@ local KeyIndex = {} KeyIndex.__index = KeyIndex -function KeyIndex.new(shared_dict, prefix, delete_callback) +function KeyIndex.new(shared_dict, lock, prefix, delete_callback) local self = setmetatable({}, KeyIndex) self.dict = shared_dict + self.lock = lock self.key_prefix = prefix .. "key_" self.delete_count = prefix .. "delete_count" self.key_count = prefix .. "key_count" @@ -19,11 +20,18 @@ function KeyIndex.new(shared_dict, prefix, delete_callback) self.keys = {} self.index = {} self.delete_callback = delete_callback + self.first_synced = false return self end -- Loads new keys that might have been added by other workers since last sync. function KeyIndex:sync() + if not self.first_synced then + local _, err = self.lock:lock("lock_key") + if err then + return + end + end local delete_count = self.dict:get(self.delete_count) or 0 local N = self.dict:get(self.key_count) or 0 if self.deleted ~= delete_count then @@ -34,6 +42,10 @@ function KeyIndex:sync() -- Sync only new keys, if there are any. self:sync_range(self.last, N) end + if not self.first_synced then + self.first_synced = true + self.lock:unlock() + end return N end @@ -82,6 +94,9 @@ function KeyIndex:add(key_or_keys, err_msg_lru_eviction) for _, key in pairs(keys) do while true do local N = self:sync() + if not self.first_synced then + return "First sync is not yet completed" + end if self.index[key] ~= nil then -- key already exists, we can skip it break