From 14256e9c4b55db7ed82f31c8a2282ed639023671 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Thu, 8 Dec 2022 14:41:25 -0800 Subject: [PATCH 01/25] temp: ngx.decode_base64url --- lua-tree/share/lua/5.1/kong/tools/utils.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/tools/utils.lua b/lua-tree/share/lua/5.1/kong/tools/utils.lua index c570bda..ef0fe0b 100644 --- a/lua-tree/share/lua/5.1/kong/tools/utils.lua +++ b/lua-tree/share/lua/5.1/kong/tools/utils.lua @@ -1267,7 +1267,7 @@ _M.yield = function() return end local try_decode_base64 do local decode_base64 = ngx.decode_base64 - local decode_base64url = require "ngx.base64".decode_base64url + local decode_base64url = ngx.decode_base64url local function decode_base64_str(str) if type(str) == "string" then @@ -1334,7 +1334,7 @@ local sha256_hex, sha256_base64, sha256_base64url do local to_hex = require "resty.string".to_hex local to_base64 = ngx.encode_base64 - local to_base64url = require "ngx.base64".encode_base64url + local to_base64url = ngx.encode_base64url local function sha256_encode(encode_alg, key) local bin, err = sha256_bin(key) From 2f294c1cc2993cddfcb2fc53c528ef7d4ec047e6 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Thu, 8 Dec 2022 15:26:00 -0800 Subject: [PATCH 02/25] replace lua resty.openssl.digest with go crypto/sha256 --- internal/vm/vm.go | 2 ++ lua-tree/share/lua/5.1/kong/tools/utils.lua | 29 +-------------------- lualibs/go/crypto/loader.go | 14 ++++++++++ lualibs/go/crypto/sha256.go | 16 ++++++++++++ 4 files changed, 33 insertions(+), 28 deletions(-) create mode 100644 lualibs/go/crypto/loader.go create mode 100644 lualibs/go/crypto/sha256.go diff --git a/internal/vm/vm.go b/internal/vm/vm.go index 669e9b1..84fe1c8 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -11,6 +11,7 @@ import ( "github.com/kong/goks" "github.com/kong/goks/internal/fs" "github.com/kong/goks/lualibs/go/cjson/safe" + "github.com/kong/goks/lualibs/go/crypto" "github.com/kong/goks/lualibs/go/ipmatcher" "github.com/kong/goks/lualibs/go/jsonschema" "github.com/kong/goks/lualibs/go/lyaml" @@ -63,6 +64,7 @@ func New(opts Opts) (*VM, error) { l.PreloadModule("go.jsonschema", jsonschema.Loader) l.PreloadModule("cjson.safe", safe.Loader) l.PreloadModule("lyaml", lyaml.Loader) + l.PreloadModule("crypto", crypto.Loader) ngx.LoadNgx(l) if err := setup(l); err != nil { diff --git a/lua-tree/share/lua/5.1/kong/tools/utils.lua b/lua-tree/share/lua/5.1/kong/tools/utils.lua index ef0fe0b..37d51ce 100644 --- a/lua-tree/share/lua/5.1/kong/tools/utils.lua +++ b/lua-tree/share/lua/5.1/kong/tools/utils.lua @@ -1299,34 +1299,7 @@ end _M.try_decode_base64 = try_decode_base64 -local sha256_bin -do - local digest = require "resty.openssl.digest" - local sha256_digest - - function sha256_bin(key) - local _, bin, err - if not sha256_digest then - sha256_digest, err = digest.new("sha256") - if err then - return nil, err - end - end - - bin, err = sha256_digest:final(key) - if err then - sha256_digest = nil - return nil, err - end - - _, err = sha256_digest:reset() - if err then - sha256_digest = nil - end - - return bin - end -end +local sha256_bin = require "crypto".sha256_bin _M.sha256_bin = sha256_bin diff --git a/lualibs/go/crypto/loader.go b/lualibs/go/crypto/loader.go new file mode 100644 index 0000000..d38dda7 --- /dev/null +++ b/lualibs/go/crypto/loader.go @@ -0,0 +1,14 @@ +package crypto + +import lua "github.com/yuin/gopher-lua" + +func Loader(l *lua.LState) int { + t := l.NewTable() + l.SetFuncs(t, api) + l.Push(t) + return 1 +} + +var api = map[string]lua.LGFunction{ + "sha256_bin": Sha256Bin, +} diff --git a/lualibs/go/crypto/sha256.go b/lualibs/go/crypto/sha256.go new file mode 100644 index 0000000..5649015 --- /dev/null +++ b/lualibs/go/crypto/sha256.go @@ -0,0 +1,16 @@ +package crypto + +import ( + "crypto/sha256" + + lua "github.com/yuin/gopher-lua" +) + +func Sha256Bin(l *lua.LState) int { + input := l.CheckString(1) + h := sha256.New() + h.Write([]byte(input)) + l.Push(lua.LString(h.Sum(nil))) + h.Reset() + return 1 +} \ No newline at end of file From 4ac298b3fd0f0f4033a9c4f7c2f506cc22550a54 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Thu, 8 Dec 2022 15:35:44 -0800 Subject: [PATCH 03/25] rename pkg crypto to sha256 --- internal/vm/vm.go | 4 ++-- lua-tree/share/lua/5.1/kong/tools/utils.lua | 2 +- lualibs/go/{crypto/sha256.go => sha256/api.go} | 2 +- lualibs/go/{crypto => sha256}/loader.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename lualibs/go/{crypto/sha256.go => sha256/api.go} (93%) rename lualibs/go/{crypto => sha256}/loader.go (93%) diff --git a/internal/vm/vm.go b/internal/vm/vm.go index 84fe1c8..dab115a 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -11,12 +11,12 @@ import ( "github.com/kong/goks" "github.com/kong/goks/internal/fs" "github.com/kong/goks/lualibs/go/cjson/safe" - "github.com/kong/goks/lualibs/go/crypto" "github.com/kong/goks/lualibs/go/ipmatcher" "github.com/kong/goks/lualibs/go/jsonschema" "github.com/kong/goks/lualibs/go/lyaml" "github.com/kong/goks/lualibs/go/ngx" "github.com/kong/goks/lualibs/go/rand" + "github.com/kong/goks/lualibs/go/sha256" "github.com/kong/goks/lualibs/go/uuid" "github.com/kong/goks/lualibs/go/x509" json "github.com/layeh/gopher-json" @@ -64,7 +64,7 @@ func New(opts Opts) (*VM, error) { l.PreloadModule("go.jsonschema", jsonschema.Loader) l.PreloadModule("cjson.safe", safe.Loader) l.PreloadModule("lyaml", lyaml.Loader) - l.PreloadModule("crypto", crypto.Loader) + l.PreloadModule("sha256", sha256.Loader) ngx.LoadNgx(l) if err := setup(l); err != nil { diff --git a/lua-tree/share/lua/5.1/kong/tools/utils.lua b/lua-tree/share/lua/5.1/kong/tools/utils.lua index 37d51ce..12aa8be 100644 --- a/lua-tree/share/lua/5.1/kong/tools/utils.lua +++ b/lua-tree/share/lua/5.1/kong/tools/utils.lua @@ -1299,7 +1299,7 @@ end _M.try_decode_base64 = try_decode_base64 -local sha256_bin = require "crypto".sha256_bin +local sha256_bin = require "sha256".sha256_bin _M.sha256_bin = sha256_bin diff --git a/lualibs/go/crypto/sha256.go b/lualibs/go/sha256/api.go similarity index 93% rename from lualibs/go/crypto/sha256.go rename to lualibs/go/sha256/api.go index 5649015..cccecd0 100644 --- a/lualibs/go/crypto/sha256.go +++ b/lualibs/go/sha256/api.go @@ -1,4 +1,4 @@ -package crypto +package sha256 import ( "crypto/sha256" diff --git a/lualibs/go/crypto/loader.go b/lualibs/go/sha256/loader.go similarity index 93% rename from lualibs/go/crypto/loader.go rename to lualibs/go/sha256/loader.go index d38dda7..f3da8ee 100644 --- a/lualibs/go/crypto/loader.go +++ b/lualibs/go/sha256/loader.go @@ -1,4 +1,4 @@ -package crypto +package sha256 import lua "github.com/yuin/gopher-lua" From b6e369b925b771b15d7ffa67995f3df399782b8b Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Thu, 8 Dec 2022 15:51:18 -0800 Subject: [PATCH 04/25] replace lua resty.string.to_hex with go encoding/hex --- internal/vm/vm.go | 2 ++ lua-tree/share/lua/5.1/kong/tools/utils.lua | 2 +- lualibs/go/hex/api.go | 20 ++++++++++++++++++++ lualibs/go/hex/loader.go | 14 ++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 lualibs/go/hex/api.go create mode 100644 lualibs/go/hex/loader.go diff --git a/internal/vm/vm.go b/internal/vm/vm.go index dab115a..f3df182 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -11,6 +11,7 @@ import ( "github.com/kong/goks" "github.com/kong/goks/internal/fs" "github.com/kong/goks/lualibs/go/cjson/safe" + "github.com/kong/goks/lualibs/go/hex" "github.com/kong/goks/lualibs/go/ipmatcher" "github.com/kong/goks/lualibs/go/jsonschema" "github.com/kong/goks/lualibs/go/lyaml" @@ -65,6 +66,7 @@ func New(opts Opts) (*VM, error) { l.PreloadModule("cjson.safe", safe.Loader) l.PreloadModule("lyaml", lyaml.Loader) l.PreloadModule("sha256", sha256.Loader) + l.PreloadModule("hex", hex.Loader) ngx.LoadNgx(l) if err := setup(l); err != nil { diff --git a/lua-tree/share/lua/5.1/kong/tools/utils.lua b/lua-tree/share/lua/5.1/kong/tools/utils.lua index 12aa8be..4d08715 100644 --- a/lua-tree/share/lua/5.1/kong/tools/utils.lua +++ b/lua-tree/share/lua/5.1/kong/tools/utils.lua @@ -1305,7 +1305,7 @@ _M.sha256_bin = sha256_bin local sha256_hex, sha256_base64, sha256_base64url do - local to_hex = require "resty.string".to_hex + local to_hex = require "hex".to_hex local to_base64 = ngx.encode_base64 local to_base64url = ngx.encode_base64url diff --git a/lualibs/go/hex/api.go b/lualibs/go/hex/api.go new file mode 100644 index 0000000..3b06a99 --- /dev/null +++ b/lualibs/go/hex/api.go @@ -0,0 +1,20 @@ +package hex + +import ( + "encoding/hex" + + lua "github.com/yuin/gopher-lua" +) + +// Encode encodes input string into hex bytes. It is a wrapper around +// encoding/hex.Encode and replaces resty.string.to_hex. +// See: https://github.com/openresty/lua-resty-string/blob/master/lib/resty/string.lua +func Encode(l *lua.LState) int { + input := l.CheckString(1) + + dst := make([]byte, hex.EncodedLen(len(input))) + hex.Encode(dst, []byte(input)) + + l.Push(lua.LString(dst)) + return 1 +} diff --git a/lualibs/go/hex/loader.go b/lualibs/go/hex/loader.go new file mode 100644 index 0000000..138e3ff --- /dev/null +++ b/lualibs/go/hex/loader.go @@ -0,0 +1,14 @@ +package hex + +import lua "github.com/yuin/gopher-lua" + +func Loader(l *lua.LState) int { + t := l.NewTable() + l.SetFuncs(t, api) + l.Push(t) + return 1 +} + +var api = map[string]lua.LGFunction{ + "to_hex": Encode, +} From 788f46c0d99bac7d11592c5a3b3f86fa17965cae Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Thu, 8 Dec 2022 16:55:01 -0800 Subject: [PATCH 05/25] replace table.nkeys with local fn --- lua-tree/share/lua/5.1/kong/db/schema/init.lua | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lua-tree/share/lua/5.1/kong/db/schema/init.lua b/lua-tree/share/lua/5.1/kong/db/schema/init.lua index 5783eb3..21e485d 100644 --- a/lua-tree/share/lua/5.1/kong/db/schema/init.lua +++ b/lua-tree/share/lua/5.1/kong/db/schema/init.lua @@ -1,7 +1,6 @@ local tablex = require "pl.tablex" local pretty = require "pl.pretty" local utils = require "kong.tools.utils" -local nkeys = require "table.nkeys" local cjson = { array_mt = {} } --- TODO(hbagdi) XXX analyze the impact local is_reference = function(value) return false end @@ -55,6 +54,23 @@ do end end +-- `table.nkeys()` is a LuaJIT function and is not available to the Lua VM that goks uses. +local nkeys +do + local ok + ok, nkeys = pcall(require, "table.nkeys") + if not ok then + nkeys = function (tab) + local count = 0 + for _, v in pairs(tab) do + if v ~= nil then + count = count + 1 + end + end + return count + end + end +end local validation_errors = { -- general message From 86df807dc3e7db638ef64a62333b9c74c031aa7f Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Thu, 8 Dec 2022 17:30:32 -0800 Subject: [PATCH 06/25] fmt and lint --- lualibs/go/sha256/api.go | 4 ++-- lualibs/go/sha256/loader.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lualibs/go/sha256/api.go b/lualibs/go/sha256/api.go index cccecd0..85c213a 100644 --- a/lualibs/go/sha256/api.go +++ b/lualibs/go/sha256/api.go @@ -6,11 +6,11 @@ import ( lua "github.com/yuin/gopher-lua" ) -func Sha256Bin(l *lua.LState) int { +func BinarySha256(l *lua.LState) int { input := l.CheckString(1) h := sha256.New() h.Write([]byte(input)) l.Push(lua.LString(h.Sum(nil))) h.Reset() return 1 -} \ No newline at end of file +} diff --git a/lualibs/go/sha256/loader.go b/lualibs/go/sha256/loader.go index f3da8ee..6be0425 100644 --- a/lualibs/go/sha256/loader.go +++ b/lualibs/go/sha256/loader.go @@ -10,5 +10,5 @@ func Loader(l *lua.LState) int { } var api = map[string]lua.LGFunction{ - "sha256_bin": Sha256Bin, + "sha256_bin": BinarySha256, } From ff8aeb1caa97562ac7e15424a00f176a48ad5f03 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Fri, 9 Dec 2022 19:43:04 -0800 Subject: [PATCH 07/25] remove log_level from constants --- lua-tree/share/lua/5.1/kong/constants.lua | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/constants.lua b/lua-tree/share/lua/5.1/kong/constants.lua index 775eae0..be9ed67 100644 --- a/lua-tree/share/lua/5.1/kong/constants.lua +++ b/lua-tree/share/lua/5.1/kong/constants.lua @@ -218,25 +218,6 @@ local constants = { KEY_FORMATS_MAP = key_formats_map, KEY_FORMATS = key_formats, - - LOG_LEVELS = { - debug = ngx.DEBUG, - info = ngx.INFO, - notice = ngx.NOTICE, - warn = ngx.WARN, - error = ngx.ERR, - crit = ngx.CRIT, - alert = ngx.ALERT, - emerg = ngx.EMERG, - [ngx.DEBUG] = "debug", - [ngx.INFO] = "info", - [ngx.NOTICE] = "notice", - [ngx.WARN] = "warn", - [ngx.ERR] = "error", - [ngx.CRIT] = "crit", - [ngx.ALERT] = "alert", - [ngx.EMERG] = "emerg", - }, } for _, v in ipairs(constants.CLUSTERING_SYNC_STATUS) do From ab748dd615dfeb2f91e8dfa04941c1caa0fbeb75 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Fri, 9 Dec 2022 19:53:58 -0800 Subject: [PATCH 08/25] replace table.new with local fn --- lua-tree/share/lua/5.1/kong/tools/uri.lua | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/tools/uri.lua b/lua-tree/share/lua/5.1/kong/tools/uri.lua index ecc5991..9a6d7e4 100644 --- a/lua-tree/share/lua/5.1/kong/tools/uri.lua +++ b/lua-tree/share/lua/5.1/kong/tools/uri.lua @@ -1,5 +1,3 @@ -local table_new = require "table.new" - local string_char = string.char local string_upper = string.upper local string_find = string.find @@ -11,6 +9,18 @@ local table_concat = table.concat local ngx_re_find = ngx.re.find local ngx_re_gsub = ngx.re.gsub +-- `table.new()` is a LuaJIT function and is not available to the Lua VM that goks uses. +local new_tab +do + local ok + ok, new_tab = pcall(require, "table.new") + if not ok then + new_tab = function(narr, nrec) + return {} + end + end +end + -- Charset: -- reserved = "!" / "*" / "'" / "(" / ")" / ";" / ":" / -- "@" / "&" / "=" / "+" / "$" / "," / "/" / "?" / "%" / "#" / "[" / "]" @@ -27,7 +37,7 @@ local ngx_re_gsub = ngx.re.gsub local RESERVED_CHARS = "!*'();:@&=+$,/?%#[]" -local chars_to_decode = table_new(256, 0) +local chars_to_decode = new_tab(256, 0) do -- reserved for i = 1, #RESERVED_CHARS do @@ -40,7 +50,7 @@ end local ESCAPE_PATTERN = "[^!#$&'()*+,/:;=?@[\\]A-Z\\d-_.~%]" -local TMP_OUTPUT = require("table.new")(16, 0) +local TMP_OUTPUT = new_tab(16, 0) local DOT = string_byte(".") local SLASH = string_byte("/") From 393daa93c18ffb3340f7481a46081c4be56eb3cf Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Fri, 9 Dec 2022 20:26:39 -0800 Subject: [PATCH 09/25] remove/replace resty.* imports --- lua-tree/share/lua/5.1/kong/router/atc.lua | 53 +++++++++++++++++----- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/router/atc.lua b/lua-tree/share/lua/5.1/kong/router/atc.lua index cd07a00..397bb64 100644 --- a/lua-tree/share/lua/5.1/kong/router/atc.lua +++ b/lua-tree/share/lua/5.1/kong/router/atc.lua @@ -2,16 +2,40 @@ local _M = {} local _MT = { __index = _M, } -local schema = require("resty.router.schema") -local router = require("resty.router.router") -local context = require("resty.router.context") -local lrucache = require("resty.lrucache") -local server_name = require("ngx.ssl").server_name -local tb_new = require("table.new") -local tb_clear = require("table.clear") +--local schema = require("resty.router.schema") +--local router = require("resty.router.router") +--local context = require("resty.router.context") +--local lrucache = require("resty.lrucache") +--local server_name = require("ngx.ssl").server_name +--local tb_new = require("table.new") +--local tb_clear = require("table.clear") local utils = require("kong.router.utils") local yield = require("kong.tools.utils").yield +-- `table.new()` is a LuaJIT function and is not available to the Lua VM that goks uses. +local new_tab +do + local ok + ok, new_tab = pcall(require, "table.new") + if not ok then + new_tab = function(narr, nrec) + return {} + end + end +end + +-- `table.clear()` is a LuaJIT function and is not available to the Lua VM that goks uses. +local clear_tab +do + local ok + ok, new_tab = pcall(require, "table.clear") + if not ok then + new_tab = function(tab) + return {} + end + end +end + local type = type local assert = assert @@ -54,7 +78,8 @@ local LOGICAL_AND = " && " -- reuse table objects -local gen_values_t = tb_new(10, 0) +--local gen_values_t = tb_new(10, 0) +local gen_values_t = new_tab(10, 0) local CACHED_SCHEMA @@ -101,7 +126,8 @@ local function gen_for_field(name, op, vals, val_transform) return nil end - tb_clear(gen_values_t) + --tb_clear(gen_values_t) + clear_tab(gen_values_t) local values_n = 0 local values = gen_values_t @@ -165,8 +191,10 @@ local function new_from_scratch(routes, get_exp_and_priority) local inst = router.new(CACHED_SCHEMA) local routes_n = #routes - local routes_t = tb_new(0, routes_n) - local services_t = tb_new(0, routes_n) + --local routes_t = tb_new(0, routes_n) + --local services_t = tb_new(0, routes_n) + local routes_t = new_tab(0, routes_n) + local services_t = new_tab(0, routes_n) local new_updated_at = 0 @@ -462,7 +490,8 @@ do local headers_t = tb_new(8, 0) get_headers_key = function(headers) - tb_clear(headers_t) + --tb_clear(headers_t) + clear_tab(headers_t) local headers_count = 0 From 739a308cf49cb5240074e5ff1a3cf6349cebca18 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Fri, 9 Dec 2022 20:43:14 -0800 Subject: [PATCH 10/25] rework goto statements --- lua-tree/share/lua/5.1/kong/pdk/node.lua | 83 +++++++++++------------- 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/pdk/node.lua b/lua-tree/share/lua/5.1/kong/pdk/node.lua index 60e6489..d487e62 100644 --- a/lua-tree/share/lua/5.1/kong/pdk/node.lua +++ b/lua-tree/share/lua/5.1/kong/pdk/node.lua @@ -169,57 +169,50 @@ local function new(self) -- get workers Lua VM allocated memory do - if not shared.kong then - goto lua_shared_dicts - end - - local keys, err = shared.kong:get_keys() - if not keys then - res.workers_lua_vms.err = "could not get kong shm keys: " .. err - goto lua_shared_dicts - end - - if #keys == 1024 then - -- Preventive warning log for future Kong developers, in case 'kong' - -- shm becomes mis-used or over-used. - ngx.log(ngx.WARN, "ngx.shared.kong:get_keys() returned 1024 keys, ", - "but it may have more") - end - - for i = 1, #keys do - local pid = match(keys[i], "kong:mem:(%d+)") - if not pid then - goto continue - end - - local w = self.table.new(0, 2) - w.pid = tonumber(pid) - - local count, err = shared.kong:get("kong:mem:" .. pid) - if err then - w.err = "could not get worker's HTTP Lua VM memory (pid: " .. - pid .. "): " .. err - - elseif type(count) ~= "number" then - w.err = "could not get worker's HTTP Lua VM memory (pid: " .. - pid .. "): reported value is corrupted" - + if shared.kong then + + local keys, err = shared.kong:get_keys() + if keys then + + if #keys == 1024 then + -- Preventive warning log for future Kong developers, in case 'kong' + -- shm becomes mis-used or over-used. + ngx.log(ngx.WARN, "ngx.shared.kong:get_keys() returned 1024 keys, ", + "but it may have more") + end + + for i = 1, #keys do + local pid = match(keys[i], "kong:mem:(%d+)") + if pid then + local w = self.table.new(0, 2) + w.pid = tonumber(pid) + + local count, err = shared.kong:get("kong:mem:" .. pid) + if err then + w.err = "could not get worker's HTTP Lua VM memory (pid: " .. + pid .. "): " .. err + + elseif type(count) ~= "number" then + w.err = "could not get worker's HTTP Lua VM memory (pid: " .. + pid .. "): reported value is corrupted" + + else + count = count * 1024 -- reported value is in kb + w.http_allocated_gc = convert_bytes(count, unit, scale) + end + + insert(res.workers_lua_vms, w) + end + end + + sort(res.workers_lua_vms, sort_pid_asc) else - count = count * 1024 -- reported value is in kb - w.http_allocated_gc = convert_bytes(count, unit, scale) + res.workers_lua_vms.err = "could not get kong shm keys: " .. err end - - insert(res.workers_lua_vms, w) - - ::continue:: end - - sort(res.workers_lua_vms, sort_pid_asc) end -- get lua_shared_dicts allocated slabs - ::lua_shared_dicts:: - for _, shm in ipairs(shms) do local allocated = shm.capacity - shm.zone:free_space() From 001809f00a4c2f2b1fb99705195fcd033897df09 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Fri, 9 Dec 2022 20:47:53 -0800 Subject: [PATCH 11/25] remove ffi imports from node.lua --- lua-tree/share/lua/5.1/kong/pdk/node.lua | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/pdk/node.lua b/lua-tree/share/lua/5.1/kong/pdk/node.lua index d487e62..db94d90 100644 --- a/lua-tree/share/lua/5.1/kong/pdk/node.lua +++ b/lua-tree/share/lua/5.1/kong/pdk/node.lua @@ -3,7 +3,6 @@ -- @module kong.node local utils = require "kong.tools.utils" -local ffi = require "ffi" local private_node = require "kong.pdk.private.node" @@ -15,9 +14,6 @@ local sort = table.sort local insert = table.insert local ngx = ngx local shared = ngx.shared -local C = ffi.C -local ffi_new = ffi.new -local ffi_str = ffi.string local NODE_ID_KEY = "kong:node_id" @@ -234,16 +230,6 @@ local function new(self) -- @usage -- local hostname = kong.node.get_hostname() function _NODE.get_hostname() - local SIZE = 253 -- max number of chars for a hostname - - local buf = ffi_new("unsigned char[?]", SIZE) - local res = C.gethostname(buf, SIZE) - - if res == 0 then - local hostname = ffi_str(buf, SIZE) - return gsub(hostname, "%z+$", "") - end - local f = io.popen("/bin/hostname") local hostname = f:read("*a") or "" f:close() From 884c12ffb69cb7c22ba9ac9e3468855d04c608fc Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Fri, 9 Dec 2022 21:25:23 -0800 Subject: [PATCH 12/25] pdk/private/node.lua remove pl imports --- .../share/lua/5.1/kong/pdk/private/node.lua | 206 +++++++++--------- lua-tree/share/lua/5.1/pl/path.lua | 1 + 2 files changed, 104 insertions(+), 103 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/pdk/private/node.lua b/lua-tree/share/lua/5.1/kong/pdk/private/node.lua index ed299c9..314cc37 100644 --- a/lua-tree/share/lua/5.1/kong/pdk/private/node.lua +++ b/lua-tree/share/lua/5.1/kong/pdk/private/node.lua @@ -1,103 +1,103 @@ -local log = require "kong.cmd.utils.log" -local utils = require "kong.tools.utils" -local pl_file = require "pl.file" -local pl_path = require "pl.path" -local pl_dir = require "pl.dir" - -local fmt = string.format - -local cached_node_id - -local function node_id_filename(prefix) - return pl_path.join(prefix, "kong.id") -end - - -local function initialize_node_id(prefix) - if not pl_path.exists(prefix) then - local ok, err = pl_dir.makepath(prefix) - if not ok then - return nil, fmt("failed to create directory %s: %s", prefix, err) - end - end - - local filename = node_id_filename(prefix) - - local file_exists = pl_path.exists(filename) - - if file_exists then - local id, err = pl_file.read(filename) - if err then - return nil, fmt("failed to access file %s: %s", filename, err) - end - - if not utils.is_valid_uuid(id) then - log.debug("file %s contains invalid uuid: %s", filename, id) - -- set false to override it when it contains an invalid uuid. - file_exists = false - end - end - - if not file_exists then - local id = utils.uuid() - log.debug("persisting node_id (%s) to %s", id, filename) - - local ok, write_err = pl_file.write(filename, id) - if not ok then - return nil, fmt("failed to persist node_id to %s: %s", filename, write_err) - end - cached_node_id = id - end - - return true -end - - -local function init_node_id(config) - if not config then - return - end - - if not config.prefix or config.role ~= "data_plane" then - return - end - - local ok, err = initialize_node_id(config.prefix) - if not ok then - log.warn(err) - end -end - - -local function load_node_id(prefix) - if not prefix then - return nil, nil - end - - if cached_node_id then - return cached_node_id, nil - end - - local filename = node_id_filename(prefix) - - if not pl_path.exists(filename) then - return nil, fmt("file %s does not exist", filename) - end - - local id, read_err = pl_file.read(filename) - if read_err then - return nil, fmt("failed to access file %s: %s", filename, read_err) - end - - if not utils.is_valid_uuid(id) then - return nil, fmt("file %s contains invalid uuid: %q", filename, id) - end - - return id, nil -end - - -return { - init_node_id = init_node_id, - load_node_id = load_node_id, -} +--local log = require "kong.cmd.utils.log" +--local utils = require "kong.tools.utils" +--local pl_file = require "pl.file" +--local pl_path = require "pl.path" +--local pl_dir = require "pl.dir" +-- +--local fmt = string.format +-- +--local cached_node_id +-- +--local function node_id_filename(prefix) +-- return pl_path.join(prefix, "kong.id") +--end +-- +-- +--local function initialize_node_id(prefix) +-- if not pl_path.exists(prefix) then +-- local ok, err = pl_dir.makepath(prefix) +-- if not ok then +-- return nil, fmt("failed to create directory %s: %s", prefix, err) +-- end +-- end +-- +-- local filename = node_id_filename(prefix) +-- +-- local file_exists = pl_path.exists(filename) +-- +-- if file_exists then +-- local id, err = pl_file.read(filename) +-- if err then +-- return nil, fmt("failed to access file %s: %s", filename, err) +-- end +-- +-- if not utils.is_valid_uuid(id) then +-- log.debug("file %s contains invalid uuid: %s", filename, id) +-- -- set false to override it when it contains an invalid uuid. +-- file_exists = false +-- end +-- end +-- +-- if not file_exists then +-- local id = utils.uuid() +-- log.debug("persisting node_id (%s) to %s", id, filename) +-- +-- local ok, write_err = pl_file.write(filename, id) +-- if not ok then +-- return nil, fmt("failed to persist node_id to %s: %s", filename, write_err) +-- end +-- cached_node_id = id +-- end +-- +-- return true +--end +-- +-- +--local function init_node_id(config) +-- if not config then +-- return +-- end +-- +-- if not config.prefix or config.role ~= "data_plane" then +-- return +-- end +-- +-- local ok, err = initialize_node_id(config.prefix) +-- if not ok then +-- log.warn(err) +-- end +--end +-- +-- +--local function load_node_id(prefix) +-- if not prefix then +-- return nil, nil +-- end +-- +-- if cached_node_id then +-- return cached_node_id, nil +-- end +-- +-- local filename = node_id_filename(prefix) +-- +-- if not pl_path.exists(filename) then +-- return nil, fmt("file %s does not exist", filename) +-- end +-- +-- local id, read_err = pl_file.read(filename) +-- if read_err then +-- return nil, fmt("failed to access file %s: %s", filename, read_err) +-- end +-- +-- if not utils.is_valid_uuid(id) then +-- return nil, fmt("file %s contains invalid uuid: %q", filename, id) +-- end +-- +-- return id, nil +--end +-- +-- +--return { +-- init_node_id = init_node_id, +-- load_node_id = load_node_id, +--} diff --git a/lua-tree/share/lua/5.1/pl/path.lua b/lua-tree/share/lua/5.1/pl/path.lua index 1e044e1..79b5261 100644 --- a/lua-tree/share/lua/5.1/pl/path.lua +++ b/lua-tree/share/lua/5.1/pl/path.lua @@ -21,6 +21,7 @@ local append, concat, remove = table.insert, table.concat, table.remove local utils = require 'pl.utils' local assert_string,raise = utils.assert_string,utils.raise +print(debug.traceback()) -- todo(aeg): remove local res,lfs = _G.pcall(_G.require,'lfs') if not res then error("pl.path requires LuaFileSystem") From 23c7b76e239f8bb0683455145fdc62446560e1da Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Fri, 9 Dec 2022 22:05:59 -0800 Subject: [PATCH 13/25] remove ngx imports from node and reports --- lua-tree/share/lua/5.1/kong/pdk/node.lua | 81 +----------- lua-tree/share/lua/5.1/kong/reports.lua | 153 ----------------------- 2 files changed, 2 insertions(+), 232 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/pdk/node.lua b/lua-tree/share/lua/5.1/kong/pdk/node.lua index db94d90..bd2d49e 100644 --- a/lua-tree/share/lua/5.1/kong/pdk/node.lua +++ b/lua-tree/share/lua/5.1/kong/pdk/node.lua @@ -12,24 +12,12 @@ local match = string.match local gsub = string.gsub local sort = table.sort local insert = table.insert -local ngx = ngx -local shared = ngx.shared local NODE_ID_KEY = "kong:node_id" local node_id local shms = {} -local n_workers = ngx.worker.count() - - -for shm_name, shm in pairs(shared) do - insert(shms, { - zone = shm, - name = shm_name, - capacity = shm:capacity(), - }) -end local function convert_bytes(bytes, unit, scale) @@ -62,23 +50,7 @@ local function new(self) return node_id end - local shm = ngx.shared.kong - - local ok, err = shm:safe_add(NODE_ID_KEY, utils.uuid()) - if not ok and err ~= "exists" then - error("failed to set 'node_id' in shm: " .. err) - end - - node_id, err = shm:get(NODE_ID_KEY) - if err then - error("failed to get 'node_id' in shm: " .. err) - end - - if not node_id then - error("no 'node_id' set in shm") - end - - return node_id + return utils.uuid() end @@ -158,56 +130,10 @@ local function new(self) end local res = { - workers_lua_vms = self.table.new(n_workers, 0), + workers_lua_vms = self.table.new(0, 0), lua_shared_dicts = self.table.new(0, #shms), } - -- get workers Lua VM allocated memory - - do - if shared.kong then - - local keys, err = shared.kong:get_keys() - if keys then - - if #keys == 1024 then - -- Preventive warning log for future Kong developers, in case 'kong' - -- shm becomes mis-used or over-used. - ngx.log(ngx.WARN, "ngx.shared.kong:get_keys() returned 1024 keys, ", - "but it may have more") - end - - for i = 1, #keys do - local pid = match(keys[i], "kong:mem:(%d+)") - if pid then - local w = self.table.new(0, 2) - w.pid = tonumber(pid) - - local count, err = shared.kong:get("kong:mem:" .. pid) - if err then - w.err = "could not get worker's HTTP Lua VM memory (pid: " .. - pid .. "): " .. err - - elseif type(count) ~= "number" then - w.err = "could not get worker's HTTP Lua VM memory (pid: " .. - pid .. "): reported value is corrupted" - - else - count = count * 1024 -- reported value is in kb - w.http_allocated_gc = convert_bytes(count, unit, scale) - end - - insert(res.workers_lua_vms, w) - end - end - - sort(res.workers_lua_vms, sort_pid_asc) - else - res.workers_lua_vms.err = "could not get kong shm keys: " .. err - end - end - end - -- get lua_shared_dicts allocated slabs for _, shm in ipairs(shms) do local allocated = shm.capacity - shm.zone:free_space() @@ -242,13 +168,10 @@ local function new(self) local id, err = private_node.load_node_id(prefix) if id then node_id = id - ngx.log(ngx.DEBUG, "restored node_id from the filesystem: ", node_id) else id = _NODE.get_id() if err then - ngx.log(ngx.WARN, "failed to restore node_id from the filesystem: ", - err, ", so a new one was generated: ", id) end end end diff --git a/lua-tree/share/lua/5.1/kong/reports.lua b/lua-tree/share/lua/5.1/kong/reports.lua index 70e3512..e62c1fc 100644 --- a/lua-tree/share/lua/5.1/kong/reports.lua +++ b/lua-tree/share/lua/5.1/kong/reports.lua @@ -6,21 +6,12 @@ local knode = (kong and kong.node) and kong.node or require "kong.pdk.node".new() -local kong_dict = ngx.shared.kong -local ngx = ngx -local tcp_sock = ngx.socket.tcp -local timer_at = ngx.timer.at -local ngx_log = ngx.log -local var = ngx.var -local subsystem = ngx.config.subsystem local concat = table.concat local tostring = tostring local lower = string.lower local pairs = pairs local error = error local type = type -local WARN = ngx.WARN -local DEBUG = ngx.DEBUG local sub = string.sub @@ -86,11 +77,6 @@ do end -local function log(lvl, ...) - ngx_log(lvl, "[reports] ", ...) -end - - local function serialize_report_value(v) if type(v) == "function" then v = v() @@ -98,9 +84,6 @@ local function serialize_report_value(v) if type(v) == "table" then local json, err = cjson.encode(v) - if err then - log(WARN, "could not JSON encode given table entity: ", err) - end v = json end @@ -142,9 +125,6 @@ local function send_report(signal_type, t, host, port) end end - local sock = tcp_sock() - sock:settimeouts(30000, 30000, 30000) - -- errors are not logged to avoid false positives for users -- who run Kong in an air-gapped environments @@ -155,7 +135,6 @@ local function send_report(signal_type, t, host, port) local hs_ok, err = sock:sslhandshake(_ssl_session, nil, _ssl_verify) if not hs_ok then - log(DEBUG, "failed to complete SSL handshake for reports: ", err) return end @@ -168,30 +147,6 @@ end -- ping timer handler - --- Hold a lock for the whole interval (exptime) to prevent multiple --- worker processes from sending the test request simultaneously. --- Other workers do not need to wait until this lock is released, --- and can ignore the event, knowing another worker is handling it. --- We subtract 1ms to the exp time to prevent a race condition --- with the next timer event. -local function get_lock(key, exptime) - local ok, err = kong_dict:safe_add(key, true, exptime - 0.001) - if not ok and err ~= "exists" then - log(WARN, "could not get lock from 'kong' shm: ", err) - end - - return ok -end - - -local function create_timer(...) - local ok, err = timer_at(...) - if not ok then - log(WARN, "could not create ping timer: ", err) - end -end - -- @param interval exposed for unit test only local function create_counter(interval) local err @@ -203,87 +158,17 @@ end local function get_counter(key) local count, err = report_counter:get(key) - if err then - log(WARN, "could not get ", key, " from 'kong' shm: ", err) - end return count or 0 end local function reset_counter(key, amount) local ok, err = report_counter:reset(key, amount) - if not ok then - log(WARN, "could not reset ", key, " in 'kong' shm: ", err) - end end local function incr_counter(key) local ok, err = report_counter:incr(key, 1) - if not ok then - log(WARN, "could not increment ", key, " in 'kong' shm: ", err) - end -end - - --- returns a string indicating the "kind" of the current request/stream: --- "http", "https", "h2c", "h2", "grpc", "grpcs", "ws", "wss", "tcp", "tls", "udp" --- or nil + error message if the suffix could not be determined -local get_current_suffix - -if subsystem == "http" then -function get_current_suffix(ctx) - local scheme = var.scheme - local proxy_mode = var.kong_proxy_mode - if scheme == "http" or scheme == "https" then - if proxy_mode == "http" or proxy_mode == "unbuffered" then - local http_upgrade = var.http_upgrade - if http_upgrade and lower(http_upgrade) == "websocket" then - if scheme == "http" then - return "ws" - end - - return "wss" - end - - if ngx.req.http_version() == 2 then - if scheme == "http" then - return "h2c" - end - - return "h2" - end - - return scheme -- http/https - end - - if proxy_mode == "grpc" then - if scheme == "http" then - return "grpc" - end - - if scheme == "https" then - return "grpcs" - end - end - end - - if ctx.KONG_UNEXPECTED then - return nil - end - - log(WARN, "could not determine log suffix (scheme=", tostring(scheme), - ", proxy_mode=", tostring(proxy_mode), ")") -end - -else -- subsystem == "stream" - function get_current_suffix(ctx) - if var.ssl_protocol then - return "tls" - end - - return lower(var.protocol) - end end @@ -294,28 +179,6 @@ local function send_ping(host, port) _ping_infos.cluster_id = kong.cluster.get_id() end - if subsystem == "stream" then - _ping_infos.streams = get_counter(STREAM_COUNT_KEY) - _ping_infos.tcp_streams = get_counter(TCP_STREAM_COUNT_KEY) - _ping_infos.udp_streams = get_counter(UDP_STREAM_COUNT_KEY) - _ping_infos.tls_streams = get_counter(TLS_STREAM_COUNT_KEY) - _ping_infos.go_plugin_reqs = get_counter(GO_PLUGINS_REQUEST_COUNT_KEY) - - _ping_infos.stream_route_cache_hit_pos = get_counter(STEAM_ROUTE_CACHE_HITS_KEY_POS) - _ping_infos.stream_route_cache_hit_neg = get_counter(STEAM_ROUTE_CACHE_HITS_KEY_NEG) - - send_report("ping", _ping_infos, host, port) - - reset_counter(STREAM_COUNT_KEY, _ping_infos.streams) - reset_counter(TCP_STREAM_COUNT_KEY, _ping_infos.tcp_streams) - reset_counter(UDP_STREAM_COUNT_KEY, _ping_infos.udp_streams) - reset_counter(TLS_STREAM_COUNT_KEY, _ping_infos.tls_streams) - reset_counter(GO_PLUGINS_REQUEST_COUNT_KEY, _ping_infos.go_plugin_reqs) - reset_counter(STEAM_ROUTE_CACHE_HITS_KEY_POS, _ping_infos.stream_route_cache_hit_pos) - reset_counter(STEAM_ROUTE_CACHE_HITS_KEY_NEG, _ping_infos.stream_route_cache_hit_neg) - return - end - _ping_infos.requests = get_counter(REQUEST_COUNT_KEY) _ping_infos.http_reqs = get_counter(HTTP_REQUEST_COUNT_KEY) _ping_infos.https_reqs = get_counter(HTTPS_REQUEST_COUNT_KEY) @@ -414,7 +277,6 @@ do local res, err = red:info("server") if type(res) ~= "string" then -- could be nil or ngx.null - ngx_log(WARN, "failed to retrieve Redis version: ", err) else -- retrieve first 2 digits only @@ -452,24 +314,9 @@ return { return end - local count_key = subsystem == "stream" and STREAM_COUNT_KEY - or REQUEST_COUNT_KEY - incr_counter(count_key) - if ctx.ran_go_plugin then incr_counter(GO_PLUGINS_REQUEST_COUNT_KEY) end - - local suffix = get_current_suffix(ctx) - if suffix then - incr_counter(count_key .. ":" .. suffix) - end - - local route_match_cached = ctx.route_match_cached - - if route_match_cached then - incr_counter(count_key .. ":" .. ROUTE_CACHE_HITS_KEY .. ":" .. route_match_cached) - end end, -- custom methods From 0345f15fa373825db79596a04195fbcb96f740fe Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Fri, 9 Dec 2022 22:09:12 -0800 Subject: [PATCH 14/25] override nkeys for router/utils --- lua-tree/share/lua/5.1/kong/router/utils.lua | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lua-tree/share/lua/5.1/kong/router/utils.lua b/lua-tree/share/lua/5.1/kong/router/utils.lua index 74576c6..c74188f 100644 --- a/lua-tree/share/lua/5.1/kong/router/utils.lua +++ b/lua-tree/share/lua/5.1/kong/router/utils.lua @@ -245,9 +245,26 @@ end local phonehome_statistics do local reports = require("kong.reports") - local nkeys = require("table.nkeys") local worker_id = ngx.worker.id + -- `table.nkeys()` is a LuaJIT function and is not available to the Lua VM that goks uses. + local nkeys + do + local ok + ok, nkeys = pcall(require, "table.nkeys") + if not ok then + nkeys = function (tab) + local count = 0 + for _, v in pairs(tab) do + if v ~= nil then + count = count + 1 + end + end + return count + end + end + end + local TILDE = byte("~") local function is_regex_magic(path) return byte(path) == TILDE From ea9bcf73253bc50539bf2b2a36b46f4d2e03c862 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Fri, 9 Dec 2022 22:11:32 -0800 Subject: [PATCH 15/25] remove ngx.worker.id ref --- lua-tree/share/lua/5.1/kong/router/utils.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/router/utils.lua b/lua-tree/share/lua/5.1/kong/router/utils.lua index c74188f..3bfee4a 100644 --- a/lua-tree/share/lua/5.1/kong/router/utils.lua +++ b/lua-tree/share/lua/5.1/kong/router/utils.lua @@ -245,7 +245,6 @@ end local phonehome_statistics do local reports = require("kong.reports") - local worker_id = ngx.worker.id -- `table.nkeys()` is a LuaJIT function and is not available to the Lua VM that goks uses. local nkeys @@ -366,7 +365,7 @@ do function phonehome_statistics(routes) local configuration = kong.configuration - if not configuration.anonymous_reports or worker_id() ~= 0 then + if not configuration.anonymous_reports then return end From b47d2e6c23ff210a2c83d022afd4319a93cb9497 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Fri, 9 Dec 2022 22:19:07 -0800 Subject: [PATCH 16/25] just get rid of atc already --- .../5.1/kong/db/schema/entities/routes.lua | 12 +- lua-tree/share/lua/5.1/kong/router/atc.lua | 1241 ++++++++--------- 2 files changed, 621 insertions(+), 632 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua b/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua index 753281a..19642f8 100644 --- a/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua +++ b/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua @@ -1,7 +1,7 @@ local typedefs = require("kong.db.schema.typedefs") local deprecation = require("kong.deprecation") -local CACHED_SCHEMA = require("kong.router.atc").schema +--local CACHED_SCHEMA = require("kong.router.atc").schema -- ATC router is incompatible with goks local kong_router_flavor = "traditional" @@ -48,12 +48,12 @@ if kong_router_flavor == "expressions" then { custom_entity_check = { field_sources = { "expression", "id", }, fn = function(entity) - local r = router.new(CACHED_SCHEMA) + --local r = router.new(CACHED_SCHEMA) - local res, err = r:add_matcher(0, entity.id, entity.expression) - if not res then - return nil, "Router Expression failed validation: " .. err - end + --local res, err = r:add_matcher(0, entity.id, entity.expression) + --if not res then + -- return nil, "Router Expression failed validation: " .. err + --end return true end, diff --git a/lua-tree/share/lua/5.1/kong/router/atc.lua b/lua-tree/share/lua/5.1/kong/router/atc.lua index 397bb64..7182e44 100644 --- a/lua-tree/share/lua/5.1/kong/router/atc.lua +++ b/lua-tree/share/lua/5.1/kong/router/atc.lua @@ -1,626 +1,615 @@ -local _M = {} -local _MT = { __index = _M, } - - ---local schema = require("resty.router.schema") ---local router = require("resty.router.router") ---local context = require("resty.router.context") ---local lrucache = require("resty.lrucache") ---local server_name = require("ngx.ssl").server_name ---local tb_new = require("table.new") ---local tb_clear = require("table.clear") -local utils = require("kong.router.utils") -local yield = require("kong.tools.utils").yield - --- `table.new()` is a LuaJIT function and is not available to the Lua VM that goks uses. -local new_tab -do - local ok - ok, new_tab = pcall(require, "table.new") - if not ok then - new_tab = function(narr, nrec) - return {} - end - end -end - --- `table.clear()` is a LuaJIT function and is not available to the Lua VM that goks uses. -local clear_tab -do - local ok - ok, new_tab = pcall(require, "table.clear") - if not ok then - new_tab = function(tab) - return {} - end - end -end - - -local type = type -local assert = assert -local setmetatable = setmetatable -local pairs = pairs -local ipairs = ipairs - - -local max = math.max -local tb_concat = table.concat -local tb_sort = table.sort - - -local ngx = ngx -local header = ngx.header -local var = ngx.var -local ngx_log = ngx.log -local get_phase = ngx.get_phase -local get_method = ngx.req.get_method -local get_headers = ngx.req.get_headers -local ngx_WARN = ngx.WARN -local ngx_ERR = ngx.ERR - - -local sanitize_uri_postfix = utils.sanitize_uri_postfix -local check_select_params = utils.check_select_params -local strip_uri_args = utils.strip_uri_args -local get_service_info = utils.get_service_info -local add_debug_headers = utils.add_debug_headers -local get_upstream_uri_v0 = utils.get_upstream_uri_v0 -local route_match_stat = utils.route_match_stat - - -local MAX_REQ_HEADERS = 100 -local DEFAULT_MATCH_LRUCACHE_SIZE = utils.DEFAULT_MATCH_LRUCACHE_SIZE - - -local LOGICAL_OR = " || " -local LOGICAL_AND = " && " - - --- reuse table objects ---local gen_values_t = tb_new(10, 0) -local gen_values_t = new_tab(10, 0) - - -local CACHED_SCHEMA -do - local str_fields = {"net.protocol", "tls.sni", - "http.method", "http.host", - "http.path", "http.raw_path", - "http.headers.*", - } - - local int_fields = {"net.port", - } - - CACHED_SCHEMA = schema.new() - - for _, v in ipairs(str_fields) do - assert(CACHED_SCHEMA:add_field(v, "String")) - end - - for _, v in ipairs(int_fields) do - assert(CACHED_SCHEMA:add_field(v, "Int")) - end -end - - -local is_empty_field -do - local null = ngx.null - local isempty = require("table.isempty") - - is_empty_field = function(f) - return f == nil or f == null or isempty(f) - end -end - - -local function escape_str(str) - return "\"" .. str:gsub([[\]], [[\\]]):gsub([["]], [[\"]]) .. "\"" -end - - -local function gen_for_field(name, op, vals, val_transform) - if is_empty_field(vals) then - return nil - end - - --tb_clear(gen_values_t) - clear_tab(gen_values_t) - - local values_n = 0 - local values = gen_values_t - - for _, p in ipairs(vals) do - values_n = values_n + 1 - local op = (type(op) == "string") and op or op(p) - values[values_n] = name .. " " .. op .. " " .. - escape_str(val_transform and val_transform(op, p) or p) - end - - if values_n > 0 then - return "(" .. tb_concat(values, LOGICAL_OR) .. ")" - end - - return nil -end - - -local function add_atc_matcher(inst, route, route_id, - get_exp_and_priority, - remove_existing) - - local exp, priority = get_exp_and_priority(route) - - if not exp then - return nil, "could not find expression, route: " .. route_id - end - - if remove_existing then - assert(inst:remove_matcher(route_id)) - end - - local ok, err = inst:add_matcher(priority, route_id, exp) - if not ok then - return nil, "could not add route: " .. route_id .. ", err: " .. err - end - - return true -end - - -local function is_http_headers_field(field) - return field:sub(1, 13) == "http.headers." -end - - -local function has_header_matching_field(fields) - for _, field in ipairs(fields) do - if is_http_headers_field(field) then - return true - end - end - - return false -end - - -local function new_from_scratch(routes, get_exp_and_priority) - local phase = get_phase() - local inst = router.new(CACHED_SCHEMA) - - local routes_n = #routes - --local routes_t = tb_new(0, routes_n) - --local services_t = tb_new(0, routes_n) - local routes_t = new_tab(0, routes_n) - local services_t = new_tab(0, routes_n) - - local new_updated_at = 0 - - for _, r in ipairs(routes) do - local route = r.route - local route_id = route.id - - if not route_id then - return nil, "could not categorize route" - end - - routes_t[route_id] = route - services_t[route_id] = r.service - - local ok, err = add_atc_matcher(inst, route, route_id, - get_exp_and_priority, false) - if ok then - new_updated_at = max(new_updated_at, route.updated_at or 0) - - else - ngx_log(ngx_ERR, err) - - routes_t[route_id] = nil - services_t[route_id] = nil - end - - yield(true, phase) - end - - local fields = inst:get_fields() - local match_headers = has_header_matching_field(fields) - - return setmetatable({ - schema = CACHED_SCHEMA, - router = inst, - routes = routes_t, - services = services_t, - fields = fields, - match_headers = match_headers, - updated_at = new_updated_at, - rebuilding = false, - }, _MT) -end - - -local function new_from_previous(routes, get_exp_and_priority, old_router) - if old_router.rebuilding then - return nil, "concurrent incremental router rebuild without mutex, this is unsafe" - end - - old_router.rebuilding = true - - local phase = get_phase() - - local inst = old_router.router - local old_routes = old_router.routes - local old_services = old_router.services - - local updated_at = old_router.updated_at - local new_updated_at = 0 - - -- create or update routes - for _, r in ipairs(routes) do - local route = r.route - local route_id = route.id - - if not route_id then - return nil, "could not categorize route" - end - - local old_route = old_routes[route_id] - local route_updated_at = route.updated_at - - route.seen = true - - old_routes[route_id] = route - old_services[route_id] = r.service - - local ok = true - local err - - if not old_route then - -- route is new - ok, err = add_atc_matcher(inst, route, route_id, get_exp_and_priority, false) - - elseif route_updated_at >= updated_at or - route_updated_at ~= old_route.updated_at then - - -- route is modified (within a sec) - ok, err = add_atc_matcher(inst, route, route_id, get_exp_and_priority, true) - end - - if ok then - new_updated_at = max(new_updated_at, route_updated_at) - - else - ngx_log(ngx_ERR, err) - - old_routes[route_id] = nil - old_services[route_id] = nil - end - - yield(true, phase) - end - - -- remove routes - for id, r in pairs(old_routes) do - if r.seen then - r.seen = nil - - else - assert(inst:remove_matcher(id)) - - old_routes[id] = nil - old_services[id] = nil - end - - yield(true, phase) - end - - local fields = inst:get_fields() - - old_router.fields = fields - old_router.match_headers = has_header_matching_field(fields) - old_router.updated_at = new_updated_at - old_router.rebuilding = false - - return old_router -end - - -function _M.new(routes, cache, cache_neg, old_router, get_exp_and_priority) - if type(routes) ~= "table" then - return error("expected arg #1 routes to be a table") - end - - local router, err - - if not old_router then - router, err = new_from_scratch(routes, get_exp_and_priority) - - else - router, err = new_from_previous(routes, get_exp_and_priority, old_router) - end - - if not router then - return nil, err - end - - router.cache = cache or lrucache.new(DEFAULT_MATCH_LRUCACHE_SIZE) - router.cache_neg = cache_neg or lrucache.new(DEFAULT_MATCH_LRUCACHE_SIZE) - - return router -end - - --- split port in host, ignore form '[...]' --- example.com:123 => example.com, 123 --- example.*:123 => example.*, 123 -local split_host_port -do - local tonumber = tonumber - local DEFAULT_HOSTS_LRUCACHE_SIZE = DEFAULT_MATCH_LRUCACHE_SIZE - - local memo_hp = lrucache.new(DEFAULT_HOSTS_LRUCACHE_SIZE) - - split_host_port = function(key) - if not key then - return nil, nil - end - - local m = memo_hp:get(key) - - if m then - return m[1], m[2] - end - - local p = key:find(":", nil, true) - if not p then - memo_hp:set(key, { key, nil }) - return key, nil - end - - local port = tonumber(key:sub(p + 1)) - - if not port then - memo_hp:set(key, { key, nil }) - return key, nil - end - - local host = key:sub(1, p - 1) - - memo_hp:set(key, { host, port }) - - return host, port - end -end - - -function _M:select(req_method, req_uri, req_host, req_scheme, - src_ip, src_port, - dst_ip, dst_port, - sni, req_headers) - - check_select_params(req_method, req_uri, req_host, req_scheme, - src_ip, src_port, - dst_ip, dst_port, - sni, req_headers) - - local c = context.new(self.schema) - - local host, port = split_host_port(req_host) - - for _, field in ipairs(self.fields) do - if field == "http.method" then - assert(c:add_value("http.method", req_method)) - - elseif field == "http.path" then - assert(c:add_value("http.path", req_uri)) - - elseif field == "http.host" then - assert(c:add_value("http.host", host)) - - elseif field == "net.port" then - assert(c:add_value("net.port", port)) - - elseif field == "net.protocol" then - assert(c:add_value("net.protocol", req_scheme)) - - elseif field == "tls.sni" then - assert(c:add_value("tls.sni", sni)) - - elseif req_headers and is_http_headers_field(field) then - local h = field:sub(14) - local v = req_headers[h] - - if v then - if type(v) == "string" then - assert(c:add_value(field, v:lower())) - - else - for _, v in ipairs(v) do - assert(c:add_value(field, v:lower())) - end - end - end - end - end - - local matched = self.router:execute(c) - if not matched then - return nil - end - - local uuid, matched_path, captures = c:get_result("http.path") - - local service = self.services[uuid] - local matched_route = self.routes[uuid] - - local service_protocol, _, --service_type - service_host, service_port, - service_hostname_type, service_path = get_service_info(service) - - local request_prefix = matched_route.strip_path and matched_path or nil - local request_postfix = request_prefix and req_uri:sub(#matched_path + 1) or req_uri:sub(2, -1) - request_postfix = sanitize_uri_postfix(request_postfix) or "" - local upstream_base = service_path or "/" - - local upstream_uri = get_upstream_uri_v0(matched_route, request_postfix, req_uri, - upstream_base) - - return { - route = matched_route, - service = service, - prefix = request_prefix, - matches = { - uri_captures = (captures and captures[1]) and captures or nil, - }, - upstream_url_t = { - type = service_hostname_type, - host = service_host, - port = service_port, - }, - upstream_scheme = service_protocol, - upstream_uri = upstream_uri, - upstream_host = matched_route.preserve_host and req_host or nil, - } -end - - -local get_headers_key -do - local headers_t = tb_new(8, 0) - - get_headers_key = function(headers) - --tb_clear(headers_t) - clear_tab(headers_t) - - local headers_count = 0 - - for name, value in pairs(headers) do - local name = name:gsub("-", "_"):lower() - - if type(value) == "table" then - for i, v in ipairs(value) do - value[i] = v:lower() - end - tb_sort(value) - value = tb_concat(value, ", ") - - else - value = value:lower() - end - - headers_t[headers_count + 1] = "|" - headers_t[headers_count + 2] = name - headers_t[headers_count + 3] = "=" - headers_t[headers_count + 4] = value - - headers_count = headers_count + 4 - end - - return tb_concat(headers_t, nil, 1, headers_count) - end -end - - -function _M:exec(ctx) - local req_method = get_method() - local req_uri = ctx and ctx.request_uri or var.request_uri - local req_host = var.http_host - local sni = server_name() - - local headers, headers_key - if self.match_headers then - local err - headers, err = get_headers(MAX_REQ_HEADERS) - if err == "truncated" then - ngx_log(ngx_WARN, "retrieved ", MAX_REQ_HEADERS, " headers for evaluation ", - "(max) but request had more; other headers will be ignored") - end - - headers["host"] = nil - - headers_key = get_headers_key(headers) - end - - req_uri = strip_uri_args(req_uri) - - -- cache lookup - - local cache_key = (req_method or "") .. "|" .. - (req_uri or "") .. "|" .. - (req_host or "") .. "|" .. - (sni or "") .. (headers_key or "") - - local match_t = self.cache:get(cache_key) - if not match_t then - if self.cache_neg:get(cache_key) then - route_match_stat(ctx, "neg") - return nil - end - - local req_scheme = ctx and ctx.scheme or var.scheme - - match_t = self:select(req_method, req_uri, req_host, req_scheme, - nil, nil, nil, nil, - sni, headers) - if not match_t then - self.cache_neg:set(cache_key, true) - return nil - end - - self.cache:set(cache_key, match_t) - - else - route_match_stat(ctx, "pos") - end - - -- found a match - - -- debug HTTP request header logic - add_debug_headers(var, header, match_t) - - return match_t -end - - -function _M._set_ngx(mock_ngx) - if type(mock_ngx) ~= "table" then - return - end - - if mock_ngx.header then - header = mock_ngx.header - end - - if mock_ngx.var then - var = mock_ngx.var - end - - if mock_ngx.log then - ngx_log = mock_ngx.log - end - - if type(mock_ngx.req) == "table" then - if mock_ngx.req.get_method then - get_method = mock_ngx.req.get_method - end - - if mock_ngx.req.get_headers then - get_headers = mock_ngx.req.get_headers - end - end -end - - -_M.schema = CACHED_SCHEMA - -_M.LOGICAL_OR = LOGICAL_OR -_M.LOGICAL_AND = LOGICAL_AND - -_M.escape_str = escape_str -_M.is_empty_field = is_empty_field -_M.gen_for_field = gen_for_field -_M.split_host_port = split_host_port - - -return _M +--local _M = {} +--local _MT = { __index = _M, } +-- +-- +----local schema = require("resty.router.schema") +----local router = require("resty.router.router") +----local context = require("resty.router.context") +----local lrucache = require("resty.lrucache") +----local server_name = require("ngx.ssl").server_name +----local tb_new = require("table.new") +----local tb_clear = require("table.clear") +--local utils = require("kong.router.utils") +--local yield = require("kong.tools.utils").yield +-- +---- `table.new()` is a LuaJIT function and is not available to the Lua VM that goks uses. +--local new_tab +--do +-- local ok +-- ok, new_tab = pcall(require, "table.new") +-- if not ok then +-- new_tab = function(narr, nrec) +-- return {} +-- end +-- end +--end +-- +---- `table.clear()` is a LuaJIT function and is not available to the Lua VM that goks uses. +--local clear_tab +--do +-- local ok +-- ok, new_tab = pcall(require, "table.clear") +-- if not ok then +-- new_tab = function(tab) +-- return {} +-- end +-- end +--end +-- +-- +--local type = type +--local assert = assert +--local setmetatable = setmetatable +--local pairs = pairs +--local ipairs = ipairs +-- +-- +--local max = math.max +--local tb_concat = table.concat +--local tb_sort = table.sort +-- +-- +--local ngx = ngx +--local var = ngx.var +--local get_phase = ngx.get_phase +--local get_method = ngx.req.get_method +--local get_headers = ngx.req.get_headers +-- +-- +--local sanitize_uri_postfix = utils.sanitize_uri_postfix +--local check_select_params = utils.check_select_params +--local strip_uri_args = utils.strip_uri_args +--local get_service_info = utils.get_service_info +--local add_debug_headers = utils.add_debug_headers +--local get_upstream_uri_v0 = utils.get_upstream_uri_v0 +--local route_match_stat = utils.route_match_stat +-- +-- +--local MAX_REQ_HEADERS = 100 +--local DEFAULT_MATCH_LRUCACHE_SIZE = utils.DEFAULT_MATCH_LRUCACHE_SIZE +-- +-- +--local LOGICAL_OR = " || " +--local LOGICAL_AND = " && " +-- +-- +---- reuse table objects +----local gen_values_t = tb_new(10, 0) +--local gen_values_t = new_tab(10, 0) +-- +-- +--local CACHED_SCHEMA +--do +-- local str_fields = {"net.protocol", "tls.sni", +-- "http.method", "http.host", +-- "http.path", "http.raw_path", +-- "http.headers.*", +-- } +-- +-- local int_fields = {"net.port", +-- } +-- +-- CACHED_SCHEMA = schema.new() +-- +-- for _, v in ipairs(str_fields) do +-- assert(CACHED_SCHEMA:add_field(v, "String")) +-- end +-- +-- for _, v in ipairs(int_fields) do +-- assert(CACHED_SCHEMA:add_field(v, "Int")) +-- end +--end +-- +-- +--local is_empty_field +--do +-- local null = ngx.null +-- local isempty = require("table.isempty") +-- +-- is_empty_field = function(f) +-- return f == nil or f == null or isempty(f) +-- end +--end +-- +-- +--local function escape_str(str) +-- return "\"" .. str:gsub([[\]], [[\\]]):gsub([["]], [[\"]]) .. "\"" +--end +-- +-- +--local function gen_for_field(name, op, vals, val_transform) +-- if is_empty_field(vals) then +-- return nil +-- end +-- +-- --tb_clear(gen_values_t) +-- clear_tab(gen_values_t) +-- +-- local values_n = 0 +-- local values = gen_values_t +-- +-- for _, p in ipairs(vals) do +-- values_n = values_n + 1 +-- local op = (type(op) == "string") and op or op(p) +-- values[values_n] = name .. " " .. op .. " " .. +-- escape_str(val_transform and val_transform(op, p) or p) +-- end +-- +-- if values_n > 0 then +-- return "(" .. tb_concat(values, LOGICAL_OR) .. ")" +-- end +-- +-- return nil +--end +-- +-- +--local function add_atc_matcher(inst, route, route_id, +-- get_exp_and_priority, +-- remove_existing) +-- +-- local exp, priority = get_exp_and_priority(route) +-- +-- if not exp then +-- return nil, "could not find expression, route: " .. route_id +-- end +-- +-- if remove_existing then +-- assert(inst:remove_matcher(route_id)) +-- end +-- +-- local ok, err = inst:add_matcher(priority, route_id, exp) +-- if not ok then +-- return nil, "could not add route: " .. route_id .. ", err: " .. err +-- end +-- +-- return true +--end +-- +-- +--local function is_http_headers_field(field) +-- return field:sub(1, 13) == "http.headers." +--end +-- +-- +--local function has_header_matching_field(fields) +-- for _, field in ipairs(fields) do +-- if is_http_headers_field(field) then +-- return true +-- end +-- end +-- +-- return false +--end +-- +-- +--local function new_from_scratch(routes, get_exp_and_priority) +-- local phase = get_phase() +-- local inst = router.new(CACHED_SCHEMA) +-- +-- local routes_n = #routes +-- --local routes_t = tb_new(0, routes_n) +-- --local services_t = tb_new(0, routes_n) +-- local routes_t = new_tab(0, routes_n) +-- local services_t = new_tab(0, routes_n) +-- +-- local new_updated_at = 0 +-- +-- for _, r in ipairs(routes) do +-- local route = r.route +-- local route_id = route.id +-- +-- if not route_id then +-- return nil, "could not categorize route" +-- end +-- +-- routes_t[route_id] = route +-- services_t[route_id] = r.service +-- +-- local ok, err = add_atc_matcher(inst, route, route_id, +-- get_exp_and_priority, false) +-- if ok then +-- new_updated_at = max(new_updated_at, route.updated_at or 0) +-- +-- else +-- +-- routes_t[route_id] = nil +-- services_t[route_id] = nil +-- end +-- +-- yield(true, phase) +-- end +-- +-- local fields = inst:get_fields() +-- local match_headers = has_header_matching_field(fields) +-- +-- return setmetatable({ +-- schema = CACHED_SCHEMA, +-- router = inst, +-- routes = routes_t, +-- services = services_t, +-- fields = fields, +-- match_headers = match_headers, +-- updated_at = new_updated_at, +-- rebuilding = false, +-- }, _MT) +--end +-- +-- +--local function new_from_previous(routes, get_exp_and_priority, old_router) +-- if old_router.rebuilding then +-- return nil, "concurrent incremental router rebuild without mutex, this is unsafe" +-- end +-- +-- old_router.rebuilding = true +-- +-- local phase = get_phase() +-- +-- local inst = old_router.router +-- local old_routes = old_router.routes +-- local old_services = old_router.services +-- +-- local updated_at = old_router.updated_at +-- local new_updated_at = 0 +-- +-- -- create or update routes +-- for _, r in ipairs(routes) do +-- local route = r.route +-- local route_id = route.id +-- +-- if not route_id then +-- return nil, "could not categorize route" +-- end +-- +-- local old_route = old_routes[route_id] +-- local route_updated_at = route.updated_at +-- +-- route.seen = true +-- +-- old_routes[route_id] = route +-- old_services[route_id] = r.service +-- +-- local ok = true +-- local err +-- +-- if not old_route then +-- -- route is new +-- ok, err = add_atc_matcher(inst, route, route_id, get_exp_and_priority, false) +-- +-- elseif route_updated_at >= updated_at or +-- route_updated_at ~= old_route.updated_at then +-- +-- -- route is modified (within a sec) +-- ok, err = add_atc_matcher(inst, route, route_id, get_exp_and_priority, true) +-- end +-- +-- if ok then +-- new_updated_at = max(new_updated_at, route_updated_at) +-- +-- else +-- +-- old_routes[route_id] = nil +-- old_services[route_id] = nil +-- end +-- +-- yield(true, phase) +-- end +-- +-- -- remove routes +-- for id, r in pairs(old_routes) do +-- if r.seen then +-- r.seen = nil +-- +-- else +-- assert(inst:remove_matcher(id)) +-- +-- old_routes[id] = nil +-- old_services[id] = nil +-- end +-- +-- yield(true, phase) +-- end +-- +-- local fields = inst:get_fields() +-- +-- old_router.fields = fields +-- old_router.match_headers = has_header_matching_field(fields) +-- old_router.updated_at = new_updated_at +-- old_router.rebuilding = false +-- +-- return old_router +--end +-- +-- +--function _M.new(routes, cache, cache_neg, old_router, get_exp_and_priority) +-- if type(routes) ~= "table" then +-- return error("expected arg #1 routes to be a table") +-- end +-- +-- local router, err +-- +-- if not old_router then +-- router, err = new_from_scratch(routes, get_exp_and_priority) +-- +-- else +-- router, err = new_from_previous(routes, get_exp_and_priority, old_router) +-- end +-- +-- if not router then +-- return nil, err +-- end +-- +-- router.cache = cache or lrucache.new(DEFAULT_MATCH_LRUCACHE_SIZE) +-- router.cache_neg = cache_neg or lrucache.new(DEFAULT_MATCH_LRUCACHE_SIZE) +-- +-- return router +--end +-- +-- +---- split port in host, ignore form '[...]' +---- example.com:123 => example.com, 123 +---- example.*:123 => example.*, 123 +--local split_host_port +--do +-- local tonumber = tonumber +-- local DEFAULT_HOSTS_LRUCACHE_SIZE = DEFAULT_MATCH_LRUCACHE_SIZE +-- +-- local memo_hp = lrucache.new(DEFAULT_HOSTS_LRUCACHE_SIZE) +-- +-- split_host_port = function(key) +-- if not key then +-- return nil, nil +-- end +-- +-- local m = memo_hp:get(key) +-- +-- if m then +-- return m[1], m[2] +-- end +-- +-- local p = key:find(":", nil, true) +-- if not p then +-- memo_hp:set(key, { key, nil }) +-- return key, nil +-- end +-- +-- local port = tonumber(key:sub(p + 1)) +-- +-- if not port then +-- memo_hp:set(key, { key, nil }) +-- return key, nil +-- end +-- +-- local host = key:sub(1, p - 1) +-- +-- memo_hp:set(key, { host, port }) +-- +-- return host, port +-- end +--end +-- +-- +--function _M:select(req_method, req_uri, req_host, req_scheme, +-- src_ip, src_port, +-- dst_ip, dst_port, +-- sni, req_headers) +-- +-- check_select_params(req_method, req_uri, req_host, req_scheme, +-- src_ip, src_port, +-- dst_ip, dst_port, +-- sni, req_headers) +-- +-- local c = context.new(self.schema) +-- +-- local host, port = split_host_port(req_host) +-- +-- for _, field in ipairs(self.fields) do +-- if field == "http.method" then +-- assert(c:add_value("http.method", req_method)) +-- +-- elseif field == "http.path" then +-- assert(c:add_value("http.path", req_uri)) +-- +-- elseif field == "http.host" then +-- assert(c:add_value("http.host", host)) +-- +-- elseif field == "net.port" then +-- assert(c:add_value("net.port", port)) +-- +-- elseif field == "net.protocol" then +-- assert(c:add_value("net.protocol", req_scheme)) +-- +-- elseif field == "tls.sni" then +-- assert(c:add_value("tls.sni", sni)) +-- +-- elseif req_headers and is_http_headers_field(field) then +-- local h = field:sub(14) +-- local v = req_headers[h] +-- +-- if v then +-- if type(v) == "string" then +-- assert(c:add_value(field, v:lower())) +-- +-- else +-- for _, v in ipairs(v) do +-- assert(c:add_value(field, v:lower())) +-- end +-- end +-- end +-- end +-- end +-- +-- local matched = self.router:execute(c) +-- if not matched then +-- return nil +-- end +-- +-- local uuid, matched_path, captures = c:get_result("http.path") +-- +-- local service = self.services[uuid] +-- local matched_route = self.routes[uuid] +-- +-- local service_protocol, _, --service_type +-- service_host, service_port, +-- service_hostname_type, service_path = get_service_info(service) +-- +-- local request_prefix = matched_route.strip_path and matched_path or nil +-- local request_postfix = request_prefix and req_uri:sub(#matched_path + 1) or req_uri:sub(2, -1) +-- request_postfix = sanitize_uri_postfix(request_postfix) or "" +-- local upstream_base = service_path or "/" +-- +-- local upstream_uri = get_upstream_uri_v0(matched_route, request_postfix, req_uri, +-- upstream_base) +-- +-- return { +-- route = matched_route, +-- service = service, +-- prefix = request_prefix, +-- matches = { +-- uri_captures = (captures and captures[1]) and captures or nil, +-- }, +-- upstream_url_t = { +-- type = service_hostname_type, +-- host = service_host, +-- port = service_port, +-- }, +-- upstream_scheme = service_protocol, +-- upstream_uri = upstream_uri, +-- upstream_host = matched_route.preserve_host and req_host or nil, +-- } +--end +-- +-- +--local get_headers_key +--do +-- local headers_t = tb_new(8, 0) +-- +-- get_headers_key = function(headers) +-- --tb_clear(headers_t) +-- clear_tab(headers_t) +-- +-- local headers_count = 0 +-- +-- for name, value in pairs(headers) do +-- local name = name:gsub("-", "_"):lower() +-- +-- if type(value) == "table" then +-- for i, v in ipairs(value) do +-- value[i] = v:lower() +-- end +-- tb_sort(value) +-- value = tb_concat(value, ", ") +-- +-- else +-- value = value:lower() +-- end +-- +-- headers_t[headers_count + 1] = "|" +-- headers_t[headers_count + 2] = name +-- headers_t[headers_count + 3] = "=" +-- headers_t[headers_count + 4] = value +-- +-- headers_count = headers_count + 4 +-- end +-- +-- return tb_concat(headers_t, nil, 1, headers_count) +-- end +--end +-- +-- +--function _M:exec(ctx) +-- local req_method = get_method() +-- local req_uri = ctx and ctx.request_uri or var.request_uri +-- local req_host = var.http_host +-- local sni = server_name() +-- +-- local headers, headers_key +-- if self.match_headers then +-- local err +-- headers, err = get_headers(MAX_REQ_HEADERS) +-- if err == "truncated" then +-- end +-- +-- headers["host"] = nil +-- +-- headers_key = get_headers_key(headers) +-- end +-- +-- req_uri = strip_uri_args(req_uri) +-- +-- -- cache lookup +-- +-- local cache_key = (req_method or "") .. "|" .. +-- (req_uri or "") .. "|" .. +-- (req_host or "") .. "|" .. +-- (sni or "") .. (headers_key or "") +-- +-- local match_t = self.cache:get(cache_key) +-- if not match_t then +-- if self.cache_neg:get(cache_key) then +-- route_match_stat(ctx, "neg") +-- return nil +-- end +-- +-- local req_scheme = ctx and ctx.scheme or var.scheme +-- +-- match_t = self:select(req_method, req_uri, req_host, req_scheme, +-- nil, nil, nil, nil, +-- sni, headers) +-- if not match_t then +-- self.cache_neg:set(cache_key, true) +-- return nil +-- end +-- +-- self.cache:set(cache_key, match_t) +-- +-- else +-- route_match_stat(ctx, "pos") +-- end +-- +-- -- found a match +-- +-- -- debug HTTP request header logic +-- +-- return match_t +--end +-- +-- +--function _M._set_ngx(mock_ngx) +-- if type(mock_ngx) ~= "table" then +-- return +-- end +-- +-- if mock_ngx.header then +-- end +-- +-- if mock_ngx.var then +-- var = mock_ngx.var +-- end +-- +-- if mock_ngx.log then +-- end +-- +-- if type(mock_ngx.req) == "table" then +-- if mock_ngx.req.get_method then +-- get_method = mock_ngx.req.get_method +-- end +-- +-- if mock_ngx.req.get_headers then +-- get_headers = mock_ngx.req.get_headers +-- end +-- end +--end +-- +-- +--_M.schema = CACHED_SCHEMA +-- +--_M.LOGICAL_OR = LOGICAL_OR +--_M.LOGICAL_AND = LOGICAL_AND +-- +--_M.escape_str = escape_str +--_M.is_empty_field = is_empty_field +--_M.gen_for_field = gen_for_field +--_M.split_host_port = split_host_port +-- +-- +--return _M From 33e4b3c61ffe7da93de7e518f3b37ea28611933c Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Mon, 12 Dec 2022 14:55:28 -0800 Subject: [PATCH 17/25] refactor: name go.hex and go.sha256 pkg --- internal/vm/vm.go | 4 ++-- lua-tree/share/lua/5.1/kong/tools/utils.lua | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/vm/vm.go b/internal/vm/vm.go index f3df182..13c3336 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -65,8 +65,8 @@ func New(opts Opts) (*VM, error) { l.PreloadModule("go.jsonschema", jsonschema.Loader) l.PreloadModule("cjson.safe", safe.Loader) l.PreloadModule("lyaml", lyaml.Loader) - l.PreloadModule("sha256", sha256.Loader) - l.PreloadModule("hex", hex.Loader) + l.PreloadModule("go.sha256", sha256.Loader) + l.PreloadModule("go.hex", hex.Loader) ngx.LoadNgx(l) if err := setup(l); err != nil { diff --git a/lua-tree/share/lua/5.1/kong/tools/utils.lua b/lua-tree/share/lua/5.1/kong/tools/utils.lua index 4d08715..929c64b 100644 --- a/lua-tree/share/lua/5.1/kong/tools/utils.lua +++ b/lua-tree/share/lua/5.1/kong/tools/utils.lua @@ -1299,13 +1299,13 @@ end _M.try_decode_base64 = try_decode_base64 -local sha256_bin = require "sha256".sha256_bin +local sha256_bin = require "go.sha256".sha256_bin _M.sha256_bin = sha256_bin local sha256_hex, sha256_base64, sha256_base64url do - local to_hex = require "hex".to_hex + local to_hex = require "go.hex".to_hex local to_base64 = ngx.encode_base64 local to_base64url = ngx.encode_base64url From 9798f2520b8e16b1bd7c60de99a43748a79c5d58 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Mon, 12 Dec 2022 15:04:35 -0800 Subject: [PATCH 18/25] empty atc and node files --- .../share/lua/5.1/kong/pdk/private/node.lua | 104 +-- lua-tree/share/lua/5.1/kong/router/atc.lua | 616 +----------------- 2 files changed, 2 insertions(+), 718 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/pdk/private/node.lua b/lua-tree/share/lua/5.1/kong/pdk/private/node.lua index 314cc37..92ece08 100644 --- a/lua-tree/share/lua/5.1/kong/pdk/private/node.lua +++ b/lua-tree/share/lua/5.1/kong/pdk/private/node.lua @@ -1,103 +1 @@ ---local log = require "kong.cmd.utils.log" ---local utils = require "kong.tools.utils" ---local pl_file = require "pl.file" ---local pl_path = require "pl.path" ---local pl_dir = require "pl.dir" --- ---local fmt = string.format --- ---local cached_node_id --- ---local function node_id_filename(prefix) --- return pl_path.join(prefix, "kong.id") ---end --- --- ---local function initialize_node_id(prefix) --- if not pl_path.exists(prefix) then --- local ok, err = pl_dir.makepath(prefix) --- if not ok then --- return nil, fmt("failed to create directory %s: %s", prefix, err) --- end --- end --- --- local filename = node_id_filename(prefix) --- --- local file_exists = pl_path.exists(filename) --- --- if file_exists then --- local id, err = pl_file.read(filename) --- if err then --- return nil, fmt("failed to access file %s: %s", filename, err) --- end --- --- if not utils.is_valid_uuid(id) then --- log.debug("file %s contains invalid uuid: %s", filename, id) --- -- set false to override it when it contains an invalid uuid. --- file_exists = false --- end --- end --- --- if not file_exists then --- local id = utils.uuid() --- log.debug("persisting node_id (%s) to %s", id, filename) --- --- local ok, write_err = pl_file.write(filename, id) --- if not ok then --- return nil, fmt("failed to persist node_id to %s: %s", filename, write_err) --- end --- cached_node_id = id --- end --- --- return true ---end --- --- ---local function init_node_id(config) --- if not config then --- return --- end --- --- if not config.prefix or config.role ~= "data_plane" then --- return --- end --- --- local ok, err = initialize_node_id(config.prefix) --- if not ok then --- log.warn(err) --- end ---end --- --- ---local function load_node_id(prefix) --- if not prefix then --- return nil, nil --- end --- --- if cached_node_id then --- return cached_node_id, nil --- end --- --- local filename = node_id_filename(prefix) --- --- if not pl_path.exists(filename) then --- return nil, fmt("file %s does not exist", filename) --- end --- --- local id, read_err = pl_file.read(filename) --- if read_err then --- return nil, fmt("failed to access file %s: %s", filename, read_err) --- end --- --- if not utils.is_valid_uuid(id) then --- return nil, fmt("file %s contains invalid uuid: %q", filename, id) --- end --- --- return id, nil ---end --- --- ---return { --- init_node_id = init_node_id, --- load_node_id = load_node_id, ---} +-- not used in koko at this time \ No newline at end of file diff --git a/lua-tree/share/lua/5.1/kong/router/atc.lua b/lua-tree/share/lua/5.1/kong/router/atc.lua index 7182e44..92ece08 100644 --- a/lua-tree/share/lua/5.1/kong/router/atc.lua +++ b/lua-tree/share/lua/5.1/kong/router/atc.lua @@ -1,615 +1 @@ ---local _M = {} ---local _MT = { __index = _M, } --- --- -----local schema = require("resty.router.schema") -----local router = require("resty.router.router") -----local context = require("resty.router.context") -----local lrucache = require("resty.lrucache") -----local server_name = require("ngx.ssl").server_name -----local tb_new = require("table.new") -----local tb_clear = require("table.clear") ---local utils = require("kong.router.utils") ---local yield = require("kong.tools.utils").yield --- ----- `table.new()` is a LuaJIT function and is not available to the Lua VM that goks uses. ---local new_tab ---do --- local ok --- ok, new_tab = pcall(require, "table.new") --- if not ok then --- new_tab = function(narr, nrec) --- return {} --- end --- end ---end --- ----- `table.clear()` is a LuaJIT function and is not available to the Lua VM that goks uses. ---local clear_tab ---do --- local ok --- ok, new_tab = pcall(require, "table.clear") --- if not ok then --- new_tab = function(tab) --- return {} --- end --- end ---end --- --- ---local type = type ---local assert = assert ---local setmetatable = setmetatable ---local pairs = pairs ---local ipairs = ipairs --- --- ---local max = math.max ---local tb_concat = table.concat ---local tb_sort = table.sort --- --- ---local ngx = ngx ---local var = ngx.var ---local get_phase = ngx.get_phase ---local get_method = ngx.req.get_method ---local get_headers = ngx.req.get_headers --- --- ---local sanitize_uri_postfix = utils.sanitize_uri_postfix ---local check_select_params = utils.check_select_params ---local strip_uri_args = utils.strip_uri_args ---local get_service_info = utils.get_service_info ---local add_debug_headers = utils.add_debug_headers ---local get_upstream_uri_v0 = utils.get_upstream_uri_v0 ---local route_match_stat = utils.route_match_stat --- --- ---local MAX_REQ_HEADERS = 100 ---local DEFAULT_MATCH_LRUCACHE_SIZE = utils.DEFAULT_MATCH_LRUCACHE_SIZE --- --- ---local LOGICAL_OR = " || " ---local LOGICAL_AND = " && " --- --- ----- reuse table objects -----local gen_values_t = tb_new(10, 0) ---local gen_values_t = new_tab(10, 0) --- --- ---local CACHED_SCHEMA ---do --- local str_fields = {"net.protocol", "tls.sni", --- "http.method", "http.host", --- "http.path", "http.raw_path", --- "http.headers.*", --- } --- --- local int_fields = {"net.port", --- } --- --- CACHED_SCHEMA = schema.new() --- --- for _, v in ipairs(str_fields) do --- assert(CACHED_SCHEMA:add_field(v, "String")) --- end --- --- for _, v in ipairs(int_fields) do --- assert(CACHED_SCHEMA:add_field(v, "Int")) --- end ---end --- --- ---local is_empty_field ---do --- local null = ngx.null --- local isempty = require("table.isempty") --- --- is_empty_field = function(f) --- return f == nil or f == null or isempty(f) --- end ---end --- --- ---local function escape_str(str) --- return "\"" .. str:gsub([[\]], [[\\]]):gsub([["]], [[\"]]) .. "\"" ---end --- --- ---local function gen_for_field(name, op, vals, val_transform) --- if is_empty_field(vals) then --- return nil --- end --- --- --tb_clear(gen_values_t) --- clear_tab(gen_values_t) --- --- local values_n = 0 --- local values = gen_values_t --- --- for _, p in ipairs(vals) do --- values_n = values_n + 1 --- local op = (type(op) == "string") and op or op(p) --- values[values_n] = name .. " " .. op .. " " .. --- escape_str(val_transform and val_transform(op, p) or p) --- end --- --- if values_n > 0 then --- return "(" .. tb_concat(values, LOGICAL_OR) .. ")" --- end --- --- return nil ---end --- --- ---local function add_atc_matcher(inst, route, route_id, --- get_exp_and_priority, --- remove_existing) --- --- local exp, priority = get_exp_and_priority(route) --- --- if not exp then --- return nil, "could not find expression, route: " .. route_id --- end --- --- if remove_existing then --- assert(inst:remove_matcher(route_id)) --- end --- --- local ok, err = inst:add_matcher(priority, route_id, exp) --- if not ok then --- return nil, "could not add route: " .. route_id .. ", err: " .. err --- end --- --- return true ---end --- --- ---local function is_http_headers_field(field) --- return field:sub(1, 13) == "http.headers." ---end --- --- ---local function has_header_matching_field(fields) --- for _, field in ipairs(fields) do --- if is_http_headers_field(field) then --- return true --- end --- end --- --- return false ---end --- --- ---local function new_from_scratch(routes, get_exp_and_priority) --- local phase = get_phase() --- local inst = router.new(CACHED_SCHEMA) --- --- local routes_n = #routes --- --local routes_t = tb_new(0, routes_n) --- --local services_t = tb_new(0, routes_n) --- local routes_t = new_tab(0, routes_n) --- local services_t = new_tab(0, routes_n) --- --- local new_updated_at = 0 --- --- for _, r in ipairs(routes) do --- local route = r.route --- local route_id = route.id --- --- if not route_id then --- return nil, "could not categorize route" --- end --- --- routes_t[route_id] = route --- services_t[route_id] = r.service --- --- local ok, err = add_atc_matcher(inst, route, route_id, --- get_exp_and_priority, false) --- if ok then --- new_updated_at = max(new_updated_at, route.updated_at or 0) --- --- else --- --- routes_t[route_id] = nil --- services_t[route_id] = nil --- end --- --- yield(true, phase) --- end --- --- local fields = inst:get_fields() --- local match_headers = has_header_matching_field(fields) --- --- return setmetatable({ --- schema = CACHED_SCHEMA, --- router = inst, --- routes = routes_t, --- services = services_t, --- fields = fields, --- match_headers = match_headers, --- updated_at = new_updated_at, --- rebuilding = false, --- }, _MT) ---end --- --- ---local function new_from_previous(routes, get_exp_and_priority, old_router) --- if old_router.rebuilding then --- return nil, "concurrent incremental router rebuild without mutex, this is unsafe" --- end --- --- old_router.rebuilding = true --- --- local phase = get_phase() --- --- local inst = old_router.router --- local old_routes = old_router.routes --- local old_services = old_router.services --- --- local updated_at = old_router.updated_at --- local new_updated_at = 0 --- --- -- create or update routes --- for _, r in ipairs(routes) do --- local route = r.route --- local route_id = route.id --- --- if not route_id then --- return nil, "could not categorize route" --- end --- --- local old_route = old_routes[route_id] --- local route_updated_at = route.updated_at --- --- route.seen = true --- --- old_routes[route_id] = route --- old_services[route_id] = r.service --- --- local ok = true --- local err --- --- if not old_route then --- -- route is new --- ok, err = add_atc_matcher(inst, route, route_id, get_exp_and_priority, false) --- --- elseif route_updated_at >= updated_at or --- route_updated_at ~= old_route.updated_at then --- --- -- route is modified (within a sec) --- ok, err = add_atc_matcher(inst, route, route_id, get_exp_and_priority, true) --- end --- --- if ok then --- new_updated_at = max(new_updated_at, route_updated_at) --- --- else --- --- old_routes[route_id] = nil --- old_services[route_id] = nil --- end --- --- yield(true, phase) --- end --- --- -- remove routes --- for id, r in pairs(old_routes) do --- if r.seen then --- r.seen = nil --- --- else --- assert(inst:remove_matcher(id)) --- --- old_routes[id] = nil --- old_services[id] = nil --- end --- --- yield(true, phase) --- end --- --- local fields = inst:get_fields() --- --- old_router.fields = fields --- old_router.match_headers = has_header_matching_field(fields) --- old_router.updated_at = new_updated_at --- old_router.rebuilding = false --- --- return old_router ---end --- --- ---function _M.new(routes, cache, cache_neg, old_router, get_exp_and_priority) --- if type(routes) ~= "table" then --- return error("expected arg #1 routes to be a table") --- end --- --- local router, err --- --- if not old_router then --- router, err = new_from_scratch(routes, get_exp_and_priority) --- --- else --- router, err = new_from_previous(routes, get_exp_and_priority, old_router) --- end --- --- if not router then --- return nil, err --- end --- --- router.cache = cache or lrucache.new(DEFAULT_MATCH_LRUCACHE_SIZE) --- router.cache_neg = cache_neg or lrucache.new(DEFAULT_MATCH_LRUCACHE_SIZE) --- --- return router ---end --- --- ----- split port in host, ignore form '[...]' ----- example.com:123 => example.com, 123 ----- example.*:123 => example.*, 123 ---local split_host_port ---do --- local tonumber = tonumber --- local DEFAULT_HOSTS_LRUCACHE_SIZE = DEFAULT_MATCH_LRUCACHE_SIZE --- --- local memo_hp = lrucache.new(DEFAULT_HOSTS_LRUCACHE_SIZE) --- --- split_host_port = function(key) --- if not key then --- return nil, nil --- end --- --- local m = memo_hp:get(key) --- --- if m then --- return m[1], m[2] --- end --- --- local p = key:find(":", nil, true) --- if not p then --- memo_hp:set(key, { key, nil }) --- return key, nil --- end --- --- local port = tonumber(key:sub(p + 1)) --- --- if not port then --- memo_hp:set(key, { key, nil }) --- return key, nil --- end --- --- local host = key:sub(1, p - 1) --- --- memo_hp:set(key, { host, port }) --- --- return host, port --- end ---end --- --- ---function _M:select(req_method, req_uri, req_host, req_scheme, --- src_ip, src_port, --- dst_ip, dst_port, --- sni, req_headers) --- --- check_select_params(req_method, req_uri, req_host, req_scheme, --- src_ip, src_port, --- dst_ip, dst_port, --- sni, req_headers) --- --- local c = context.new(self.schema) --- --- local host, port = split_host_port(req_host) --- --- for _, field in ipairs(self.fields) do --- if field == "http.method" then --- assert(c:add_value("http.method", req_method)) --- --- elseif field == "http.path" then --- assert(c:add_value("http.path", req_uri)) --- --- elseif field == "http.host" then --- assert(c:add_value("http.host", host)) --- --- elseif field == "net.port" then --- assert(c:add_value("net.port", port)) --- --- elseif field == "net.protocol" then --- assert(c:add_value("net.protocol", req_scheme)) --- --- elseif field == "tls.sni" then --- assert(c:add_value("tls.sni", sni)) --- --- elseif req_headers and is_http_headers_field(field) then --- local h = field:sub(14) --- local v = req_headers[h] --- --- if v then --- if type(v) == "string" then --- assert(c:add_value(field, v:lower())) --- --- else --- for _, v in ipairs(v) do --- assert(c:add_value(field, v:lower())) --- end --- end --- end --- end --- end --- --- local matched = self.router:execute(c) --- if not matched then --- return nil --- end --- --- local uuid, matched_path, captures = c:get_result("http.path") --- --- local service = self.services[uuid] --- local matched_route = self.routes[uuid] --- --- local service_protocol, _, --service_type --- service_host, service_port, --- service_hostname_type, service_path = get_service_info(service) --- --- local request_prefix = matched_route.strip_path and matched_path or nil --- local request_postfix = request_prefix and req_uri:sub(#matched_path + 1) or req_uri:sub(2, -1) --- request_postfix = sanitize_uri_postfix(request_postfix) or "" --- local upstream_base = service_path or "/" --- --- local upstream_uri = get_upstream_uri_v0(matched_route, request_postfix, req_uri, --- upstream_base) --- --- return { --- route = matched_route, --- service = service, --- prefix = request_prefix, --- matches = { --- uri_captures = (captures and captures[1]) and captures or nil, --- }, --- upstream_url_t = { --- type = service_hostname_type, --- host = service_host, --- port = service_port, --- }, --- upstream_scheme = service_protocol, --- upstream_uri = upstream_uri, --- upstream_host = matched_route.preserve_host and req_host or nil, --- } ---end --- --- ---local get_headers_key ---do --- local headers_t = tb_new(8, 0) --- --- get_headers_key = function(headers) --- --tb_clear(headers_t) --- clear_tab(headers_t) --- --- local headers_count = 0 --- --- for name, value in pairs(headers) do --- local name = name:gsub("-", "_"):lower() --- --- if type(value) == "table" then --- for i, v in ipairs(value) do --- value[i] = v:lower() --- end --- tb_sort(value) --- value = tb_concat(value, ", ") --- --- else --- value = value:lower() --- end --- --- headers_t[headers_count + 1] = "|" --- headers_t[headers_count + 2] = name --- headers_t[headers_count + 3] = "=" --- headers_t[headers_count + 4] = value --- --- headers_count = headers_count + 4 --- end --- --- return tb_concat(headers_t, nil, 1, headers_count) --- end ---end --- --- ---function _M:exec(ctx) --- local req_method = get_method() --- local req_uri = ctx and ctx.request_uri or var.request_uri --- local req_host = var.http_host --- local sni = server_name() --- --- local headers, headers_key --- if self.match_headers then --- local err --- headers, err = get_headers(MAX_REQ_HEADERS) --- if err == "truncated" then --- end --- --- headers["host"] = nil --- --- headers_key = get_headers_key(headers) --- end --- --- req_uri = strip_uri_args(req_uri) --- --- -- cache lookup --- --- local cache_key = (req_method or "") .. "|" .. --- (req_uri or "") .. "|" .. --- (req_host or "") .. "|" .. --- (sni or "") .. (headers_key or "") --- --- local match_t = self.cache:get(cache_key) --- if not match_t then --- if self.cache_neg:get(cache_key) then --- route_match_stat(ctx, "neg") --- return nil --- end --- --- local req_scheme = ctx and ctx.scheme or var.scheme --- --- match_t = self:select(req_method, req_uri, req_host, req_scheme, --- nil, nil, nil, nil, --- sni, headers) --- if not match_t then --- self.cache_neg:set(cache_key, true) --- return nil --- end --- --- self.cache:set(cache_key, match_t) --- --- else --- route_match_stat(ctx, "pos") --- end --- --- -- found a match --- --- -- debug HTTP request header logic --- --- return match_t ---end --- --- ---function _M._set_ngx(mock_ngx) --- if type(mock_ngx) ~= "table" then --- return --- end --- --- if mock_ngx.header then --- end --- --- if mock_ngx.var then --- var = mock_ngx.var --- end --- --- if mock_ngx.log then --- end --- --- if type(mock_ngx.req) == "table" then --- if mock_ngx.req.get_method then --- get_method = mock_ngx.req.get_method --- end --- --- if mock_ngx.req.get_headers then --- get_headers = mock_ngx.req.get_headers --- end --- end ---end --- --- ---_M.schema = CACHED_SCHEMA --- ---_M.LOGICAL_OR = LOGICAL_OR ---_M.LOGICAL_AND = LOGICAL_AND --- ---_M.escape_str = escape_str ---_M.is_empty_field = is_empty_field ---_M.gen_for_field = gen_for_field ---_M.split_host_port = split_host_port --- --- ---return _M +-- not used in koko at this time \ No newline at end of file From 8bd2cdc3e54e134da7600e827de70759a64cf9b4 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Mon, 12 Dec 2022 15:06:34 -0800 Subject: [PATCH 19/25] remove commented code --- .../share/lua/5.1/kong/db/schema/entities/routes.lua | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua b/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua index 19642f8..389d7dc 100644 --- a/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua +++ b/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua @@ -1,8 +1,6 @@ local typedefs = require("kong.db.schema.typedefs") local deprecation = require("kong.deprecation") ---local CACHED_SCHEMA = require("kong.router.atc").schema - -- ATC router is incompatible with goks local kong_router_flavor = "traditional" @@ -48,13 +46,6 @@ if kong_router_flavor == "expressions" then { custom_entity_check = { field_sources = { "expression", "id", }, fn = function(entity) - --local r = router.new(CACHED_SCHEMA) - - --local res, err = r:add_matcher(0, entity.id, entity.expression) - --if not res then - -- return nil, "Router Expression failed validation: " .. err - --end - return true end, } }, From a7b66b9809e0bcd03375e13354149887c446c739 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Mon, 12 Dec 2022 15:32:29 -0800 Subject: [PATCH 20/25] remove debug traceback --- lua-tree/share/lua/5.1/pl/path.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/lua-tree/share/lua/5.1/pl/path.lua b/lua-tree/share/lua/5.1/pl/path.lua index 79b5261..1e044e1 100644 --- a/lua-tree/share/lua/5.1/pl/path.lua +++ b/lua-tree/share/lua/5.1/pl/path.lua @@ -21,7 +21,6 @@ local append, concat, remove = table.insert, table.concat, table.remove local utils = require 'pl.utils' local assert_string,raise = utils.assert_string,utils.raise -print(debug.traceback()) -- todo(aeg): remove local res,lfs = _G.pcall(_G.require,'lfs') if not res then error("pl.path requires LuaFileSystem") From 74a909b3634d20967d390e18433d11b165e98dfa Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Tue, 13 Dec 2022 11:47:24 -0800 Subject: [PATCH 21/25] move new_tab and nkeys to kong.tools.utils --- .../share/lua/5.1/kong/db/schema/init.lua | 32 ++----------------- lua-tree/share/lua/5.1/kong/router/utils.lua | 19 +---------- lua-tree/share/lua/5.1/kong/tools/uri.lua | 18 +++-------- lua-tree/share/lua/5.1/kong/tools/utils.lua | 32 +++++++++++++++++++ 4 files changed, 39 insertions(+), 62 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/db/schema/init.lua b/lua-tree/share/lua/5.1/kong/db/schema/init.lua index 21e485d..41cf3b3 100644 --- a/lua-tree/share/lua/5.1/kong/db/schema/init.lua +++ b/lua-tree/share/lua/5.1/kong/db/schema/init.lua @@ -1,6 +1,8 @@ local tablex = require "pl.tablex" local pretty = require "pl.pretty" local utils = require "kong.tools.utils" +local nkeys = utils.nkeys +local new_tab = utils.new_tab local cjson = { array_mt = {} } --- TODO(hbagdi) XXX analyze the impact local is_reference = function(value) return false end @@ -42,36 +44,6 @@ Schema.__index = Schema local _cache = {} local _workspaceable = {} --- `table.new()` is a LuaJIT function and is not available to the Lua VM that goks uses. -local new_tab -do - local ok - ok, new_tab = pcall(require, "table.new") - if not ok then - new_tab = function(narr, nrec) - return {} - end - end -end - --- `table.nkeys()` is a LuaJIT function and is not available to the Lua VM that goks uses. -local nkeys -do - local ok - ok, nkeys = pcall(require, "table.nkeys") - if not ok then - nkeys = function (tab) - local count = 0 - for _, v in pairs(tab) do - if v ~= nil then - count = count + 1 - end - end - return count - end - end -end - local validation_errors = { -- general message ERROR = "Validation error: %s", diff --git a/lua-tree/share/lua/5.1/kong/router/utils.lua b/lua-tree/share/lua/5.1/kong/router/utils.lua index 3bfee4a..101fd9c 100644 --- a/lua-tree/share/lua/5.1/kong/router/utils.lua +++ b/lua-tree/share/lua/5.1/kong/router/utils.lua @@ -245,24 +245,7 @@ end local phonehome_statistics do local reports = require("kong.reports") - - -- `table.nkeys()` is a LuaJIT function and is not available to the Lua VM that goks uses. - local nkeys - do - local ok - ok, nkeys = pcall(require, "table.nkeys") - if not ok then - nkeys = function (tab) - local count = 0 - for _, v in pairs(tab) do - if v ~= nil then - count = count + 1 - end - end - return count - end - end - end + local nkeys = require("kong.tools.utils").nkeys local TILDE = byte("~") local function is_regex_magic(path) diff --git a/lua-tree/share/lua/5.1/kong/tools/uri.lua b/lua-tree/share/lua/5.1/kong/tools/uri.lua index 9a6d7e4..e8ae945 100644 --- a/lua-tree/share/lua/5.1/kong/tools/uri.lua +++ b/lua-tree/share/lua/5.1/kong/tools/uri.lua @@ -1,3 +1,5 @@ +local table_new = require "kong.tools.utils".new_tab + local string_char = string.char local string_upper = string.upper local string_find = string.find @@ -9,18 +11,6 @@ local table_concat = table.concat local ngx_re_find = ngx.re.find local ngx_re_gsub = ngx.re.gsub --- `table.new()` is a LuaJIT function and is not available to the Lua VM that goks uses. -local new_tab -do - local ok - ok, new_tab = pcall(require, "table.new") - if not ok then - new_tab = function(narr, nrec) - return {} - end - end -end - -- Charset: -- reserved = "!" / "*" / "'" / "(" / ")" / ";" / ":" / -- "@" / "&" / "=" / "+" / "$" / "," / "/" / "?" / "%" / "#" / "[" / "]" @@ -37,7 +27,7 @@ end local RESERVED_CHARS = "!*'();:@&=+$,/?%#[]" -local chars_to_decode = new_tab(256, 0) +local chars_to_decode = table_new(256, 0) do -- reserved for i = 1, #RESERVED_CHARS do @@ -50,7 +40,7 @@ end local ESCAPE_PATTERN = "[^!#$&'()*+,/:;=?@[\\]A-Z\\d-_.~%]" -local TMP_OUTPUT = new_tab(16, 0) +local TMP_OUTPUT = table_new(16, 0) local DOT = string_byte(".") local SLASH = string_byte("/") diff --git a/lua-tree/share/lua/5.1/kong/tools/utils.lua b/lua-tree/share/lua/5.1/kong/tools/utils.lua index 929c64b..375970d 100644 --- a/lua-tree/share/lua/5.1/kong/tools/utils.lua +++ b/lua-tree/share/lua/5.1/kong/tools/utils.lua @@ -1334,4 +1334,36 @@ _M.sha256_hex = sha256_hex _M.sha256_base64 = sha256_base64 _M.sha256_base64url = sha256_base64url +-- `table.new()` is a LuaJIT function and is not available to the Lua VM that goks uses. +local new_tab +do + local ok + ok, new_tab = pcall(require, "table.new") + if not ok then + new_tab = function(narr, nrec) + return {} + end + end +end + +-- `table.nkeys()` is a LuaJIT function and is not available to the Lua VM that goks uses. +local nkeys +do + local ok + ok, nkeys = pcall(require, "table.nkeys") + if not ok then + nkeys = function (tab) + local count = 0 + for _, v in pairs(tab) do + if v ~= nil then + count = count + 1 + end + end + return count + end + end +end +_M.new_tab = new_tab +_M.nkeys = nkeys + return _M From fc044a8bcccf283a945464494cc4d46387c3db60 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Tue, 13 Dec 2022 11:49:58 -0800 Subject: [PATCH 22/25] remove node, add newlines --- lua-tree/share/lua/5.1/kong/pdk/node.lua | 186 +----------------- .../share/lua/5.1/kong/pdk/private/node.lua | 2 +- lua-tree/share/lua/5.1/kong/router/atc.lua | 2 +- 3 files changed, 3 insertions(+), 187 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/pdk/node.lua b/lua-tree/share/lua/5.1/kong/pdk/node.lua index bd2d49e..85fc2fd 100644 --- a/lua-tree/share/lua/5.1/kong/pdk/node.lua +++ b/lua-tree/share/lua/5.1/kong/pdk/node.lua @@ -1,185 +1 @@ ---- Node-level utilities. --- --- @module kong.node - -local utils = require "kong.tools.utils" -local private_node = require "kong.pdk.private.node" - - -local floor = math.floor -local lower = string.lower -local match = string.match -local gsub = string.gsub -local sort = table.sort -local insert = table.insert - -local NODE_ID_KEY = "kong:node_id" - - -local node_id -local shms = {} - - -local function convert_bytes(bytes, unit, scale) - if not unit or lower(unit) == "b" then - return floor(bytes) - end - - return utils.bytes_to_str(bytes, unit, scale) -end - - -local function sort_pid_asc(a, b) - return a.pid < b.pid -end - - -local function new(self) - local _NODE = {} - - - --- - -- Returns the ID used by this node to describe itself. - -- - -- @function kong.node.get_id - -- @treturn string The v4 UUID used by this node as its ID. - -- @usage - -- local id = kong.node.get_id() - function _NODE.get_id() - if node_id then - return node_id - end - - return utils.uuid() - end - - - --- - -- Returns memory usage statistics about this node. - -- - -- @function kong.node.get_memory_stats - -- @tparam[opt] string unit The unit that memory is reported in. Can be - -- any of `b/B`, `k/K`, `m/M`, or `g/G` for bytes, kibibytes, mebibytes, - -- or gibibytes, respectively. Defaults to `b` (bytes). - -- @tparam[opt] number scale The number of digits to the right of the decimal - -- point. Defaults to 2. - -- @treturn table A table containing memory usage statistics for this node. - -- If `unit` is `b/B` (the default), reported values are Lua numbers. - -- Otherwise, reported values are strings with the unit as a suffix. - -- @usage - -- local res = kong.node.get_memory_stats() - -- -- res will have the following structure: - -- { - -- lua_shared_dicts = { - -- kong = { - -- allocated_slabs = 12288, - -- capacity = 24576 - -- }, - -- kong_db_cache = { - -- allocated_slabs = 12288, - -- capacity = 12288 - -- } - -- }, - -- workers_lua_vms = { - -- { - -- http_allocated_gc = 1102, - -- pid = 18004 - -- }, - -- { - -- http_allocated_gc = 1102, - -- pid = 18005 - -- } - -- } - -- } - -- - -- local res = kong.node.get_memory_stats("k", 1) - -- -- res will have the following structure: - -- { - -- lua_shared_dicts = { - -- kong = { - -- allocated_slabs = "12.0 KiB", - -- capacity = "24.0 KiB", - -- }, - -- kong_db_cache = { - -- allocated_slabs = "12.0 KiB", - -- capacity = "12.0 KiB", - -- } - -- }, - -- workers_lua_vms = { - -- { - -- http_allocated_gc = "1.1 KiB", - -- pid = 18004 - -- }, - -- { - -- http_allocated_gc = "1.1 KiB", - -- pid = 18005 - -- } - -- } - -- } - function _NODE.get_memory_stats(unit, scale) - -- validate arguments - - do - unit = unit or "b" - scale = scale or 2 - - local pok, perr = pcall(utils.bytes_to_str, 0, unit, scale) - if not pok then - error(perr, 2) - end - end - - local res = { - workers_lua_vms = self.table.new(0, 0), - lua_shared_dicts = self.table.new(0, #shms), - } - - -- get lua_shared_dicts allocated slabs - for _, shm in ipairs(shms) do - local allocated = shm.capacity - shm.zone:free_space() - - res.lua_shared_dicts[shm.name] = { - capacity = convert_bytes(shm.capacity, unit, scale), - allocated_slabs = convert_bytes(allocated, unit, scale), - } - end - - return res - end - - - --- - -- Returns the name used by the local machine. - -- - -- @function kong.node.get_hostname - -- @treturn string The local machine hostname. - -- @usage - -- local hostname = kong.node.get_hostname() - function _NODE.get_hostname() - local f = io.popen("/bin/hostname") - local hostname = f:read("*a") or "" - f:close() - return gsub(hostname, "\n$", "") - end - - - local prefix = self and self.configuration and self.configuration.prefix - if prefix and self.configuration.role == "data_plane" then - local id, err = private_node.load_node_id(prefix) - if id then - node_id = id - - else - id = _NODE.get_id() - if err then - end - end - end - - return _NODE -end - - -return { - new = new, -} +-- not used in koko at this time diff --git a/lua-tree/share/lua/5.1/kong/pdk/private/node.lua b/lua-tree/share/lua/5.1/kong/pdk/private/node.lua index 92ece08..85fc2fd 100644 --- a/lua-tree/share/lua/5.1/kong/pdk/private/node.lua +++ b/lua-tree/share/lua/5.1/kong/pdk/private/node.lua @@ -1 +1 @@ --- not used in koko at this time \ No newline at end of file +-- not used in koko at this time diff --git a/lua-tree/share/lua/5.1/kong/router/atc.lua b/lua-tree/share/lua/5.1/kong/router/atc.lua index 92ece08..85fc2fd 100644 --- a/lua-tree/share/lua/5.1/kong/router/atc.lua +++ b/lua-tree/share/lua/5.1/kong/router/atc.lua @@ -1 +1 @@ --- not used in koko at this time \ No newline at end of file +-- not used in koko at this time From cddf098143aaa947e37428dceb5614e290114c44 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Tue, 13 Dec 2022 11:57:22 -0800 Subject: [PATCH 23/25] revert reports changes --- lua-tree/share/lua/5.1/kong/reports.lua | 153 ++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/lua-tree/share/lua/5.1/kong/reports.lua b/lua-tree/share/lua/5.1/kong/reports.lua index e62c1fc..70e3512 100644 --- a/lua-tree/share/lua/5.1/kong/reports.lua +++ b/lua-tree/share/lua/5.1/kong/reports.lua @@ -6,12 +6,21 @@ local knode = (kong and kong.node) and kong.node or require "kong.pdk.node".new() +local kong_dict = ngx.shared.kong +local ngx = ngx +local tcp_sock = ngx.socket.tcp +local timer_at = ngx.timer.at +local ngx_log = ngx.log +local var = ngx.var +local subsystem = ngx.config.subsystem local concat = table.concat local tostring = tostring local lower = string.lower local pairs = pairs local error = error local type = type +local WARN = ngx.WARN +local DEBUG = ngx.DEBUG local sub = string.sub @@ -77,6 +86,11 @@ do end +local function log(lvl, ...) + ngx_log(lvl, "[reports] ", ...) +end + + local function serialize_report_value(v) if type(v) == "function" then v = v() @@ -84,6 +98,9 @@ local function serialize_report_value(v) if type(v) == "table" then local json, err = cjson.encode(v) + if err then + log(WARN, "could not JSON encode given table entity: ", err) + end v = json end @@ -125,6 +142,9 @@ local function send_report(signal_type, t, host, port) end end + local sock = tcp_sock() + sock:settimeouts(30000, 30000, 30000) + -- errors are not logged to avoid false positives for users -- who run Kong in an air-gapped environments @@ -135,6 +155,7 @@ local function send_report(signal_type, t, host, port) local hs_ok, err = sock:sslhandshake(_ssl_session, nil, _ssl_verify) if not hs_ok then + log(DEBUG, "failed to complete SSL handshake for reports: ", err) return end @@ -147,6 +168,30 @@ end -- ping timer handler + +-- Hold a lock for the whole interval (exptime) to prevent multiple +-- worker processes from sending the test request simultaneously. +-- Other workers do not need to wait until this lock is released, +-- and can ignore the event, knowing another worker is handling it. +-- We subtract 1ms to the exp time to prevent a race condition +-- with the next timer event. +local function get_lock(key, exptime) + local ok, err = kong_dict:safe_add(key, true, exptime - 0.001) + if not ok and err ~= "exists" then + log(WARN, "could not get lock from 'kong' shm: ", err) + end + + return ok +end + + +local function create_timer(...) + local ok, err = timer_at(...) + if not ok then + log(WARN, "could not create ping timer: ", err) + end +end + -- @param interval exposed for unit test only local function create_counter(interval) local err @@ -158,17 +203,87 @@ end local function get_counter(key) local count, err = report_counter:get(key) + if err then + log(WARN, "could not get ", key, " from 'kong' shm: ", err) + end return count or 0 end local function reset_counter(key, amount) local ok, err = report_counter:reset(key, amount) + if not ok then + log(WARN, "could not reset ", key, " in 'kong' shm: ", err) + end end local function incr_counter(key) local ok, err = report_counter:incr(key, 1) + if not ok then + log(WARN, "could not increment ", key, " in 'kong' shm: ", err) + end +end + + +-- returns a string indicating the "kind" of the current request/stream: +-- "http", "https", "h2c", "h2", "grpc", "grpcs", "ws", "wss", "tcp", "tls", "udp" +-- or nil + error message if the suffix could not be determined +local get_current_suffix + +if subsystem == "http" then +function get_current_suffix(ctx) + local scheme = var.scheme + local proxy_mode = var.kong_proxy_mode + if scheme == "http" or scheme == "https" then + if proxy_mode == "http" or proxy_mode == "unbuffered" then + local http_upgrade = var.http_upgrade + if http_upgrade and lower(http_upgrade) == "websocket" then + if scheme == "http" then + return "ws" + end + + return "wss" + end + + if ngx.req.http_version() == 2 then + if scheme == "http" then + return "h2c" + end + + return "h2" + end + + return scheme -- http/https + end + + if proxy_mode == "grpc" then + if scheme == "http" then + return "grpc" + end + + if scheme == "https" then + return "grpcs" + end + end + end + + if ctx.KONG_UNEXPECTED then + return nil + end + + log(WARN, "could not determine log suffix (scheme=", tostring(scheme), + ", proxy_mode=", tostring(proxy_mode), ")") +end + +else -- subsystem == "stream" + function get_current_suffix(ctx) + if var.ssl_protocol then + return "tls" + end + + return lower(var.protocol) + end end @@ -179,6 +294,28 @@ local function send_ping(host, port) _ping_infos.cluster_id = kong.cluster.get_id() end + if subsystem == "stream" then + _ping_infos.streams = get_counter(STREAM_COUNT_KEY) + _ping_infos.tcp_streams = get_counter(TCP_STREAM_COUNT_KEY) + _ping_infos.udp_streams = get_counter(UDP_STREAM_COUNT_KEY) + _ping_infos.tls_streams = get_counter(TLS_STREAM_COUNT_KEY) + _ping_infos.go_plugin_reqs = get_counter(GO_PLUGINS_REQUEST_COUNT_KEY) + + _ping_infos.stream_route_cache_hit_pos = get_counter(STEAM_ROUTE_CACHE_HITS_KEY_POS) + _ping_infos.stream_route_cache_hit_neg = get_counter(STEAM_ROUTE_CACHE_HITS_KEY_NEG) + + send_report("ping", _ping_infos, host, port) + + reset_counter(STREAM_COUNT_KEY, _ping_infos.streams) + reset_counter(TCP_STREAM_COUNT_KEY, _ping_infos.tcp_streams) + reset_counter(UDP_STREAM_COUNT_KEY, _ping_infos.udp_streams) + reset_counter(TLS_STREAM_COUNT_KEY, _ping_infos.tls_streams) + reset_counter(GO_PLUGINS_REQUEST_COUNT_KEY, _ping_infos.go_plugin_reqs) + reset_counter(STEAM_ROUTE_CACHE_HITS_KEY_POS, _ping_infos.stream_route_cache_hit_pos) + reset_counter(STEAM_ROUTE_CACHE_HITS_KEY_NEG, _ping_infos.stream_route_cache_hit_neg) + return + end + _ping_infos.requests = get_counter(REQUEST_COUNT_KEY) _ping_infos.http_reqs = get_counter(HTTP_REQUEST_COUNT_KEY) _ping_infos.https_reqs = get_counter(HTTPS_REQUEST_COUNT_KEY) @@ -277,6 +414,7 @@ do local res, err = red:info("server") if type(res) ~= "string" then -- could be nil or ngx.null + ngx_log(WARN, "failed to retrieve Redis version: ", err) else -- retrieve first 2 digits only @@ -314,9 +452,24 @@ return { return end + local count_key = subsystem == "stream" and STREAM_COUNT_KEY + or REQUEST_COUNT_KEY + incr_counter(count_key) + if ctx.ran_go_plugin then incr_counter(GO_PLUGINS_REQUEST_COUNT_KEY) end + + local suffix = get_current_suffix(ctx) + if suffix then + incr_counter(count_key .. ":" .. suffix) + end + + local route_match_cached = ctx.route_match_cached + + if route_match_cached then + incr_counter(count_key .. ":" .. ROUTE_CACHE_HITS_KEY .. ":" .. route_match_cached) + end end, -- custom methods From 821f5f70ad55b951b34cd620b43487f4e63a5f45 Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Tue, 13 Dec 2022 12:17:26 -0800 Subject: [PATCH 24/25] revert router.utils changes --- lua-tree/share/lua/5.1/kong/router/utils.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lua-tree/share/lua/5.1/kong/router/utils.lua b/lua-tree/share/lua/5.1/kong/router/utils.lua index 101fd9c..74576c6 100644 --- a/lua-tree/share/lua/5.1/kong/router/utils.lua +++ b/lua-tree/share/lua/5.1/kong/router/utils.lua @@ -245,7 +245,8 @@ end local phonehome_statistics do local reports = require("kong.reports") - local nkeys = require("kong.tools.utils").nkeys + local nkeys = require("table.nkeys") + local worker_id = ngx.worker.id local TILDE = byte("~") local function is_regex_magic(path) @@ -348,7 +349,7 @@ do function phonehome_statistics(routes) local configuration = kong.configuration - if not configuration.anonymous_reports then + if not configuration.anonymous_reports or worker_id() ~= 0 then return end From 4c25d0044a0a6795cacaeaa1bb18024b8483c28b Mon Sep 17 00:00:00 2001 From: Alex Gaesser Date: Wed, 14 Dec 2022 12:21:38 -0800 Subject: [PATCH 25/25] update patchfile --- patches/lua-tree.patch | 1232 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 1206 insertions(+), 26 deletions(-) diff --git a/patches/lua-tree.patch b/patches/lua-tree.patch index bb459b9..a2cc771 100644 --- a/patches/lua-tree.patch +++ b/patches/lua-tree.patch @@ -48,6 +48,7 @@ schema/init.lua: - remove log statements - ensure custom_entity_checks in schemas return error message - Reintroducing fallback `new_tab()` function, as `table.new()` LuaJIT function is not available for use. +- Fallback `nkeys()` function, as `table.nkeys` LuaJIT function is not available for use. schema/typedefs.lua: @@ -72,6 +73,8 @@ tools/utils.lua: - Use go.ipmatcher instead of resty.ipmatcher - Reintroducing fallback `is_array_fast()` function, as `table.isarray()` OpenResty function is not available for use. - Reintroducing fallback `clone()` function, as `table.clone()` OpenResty function is not available for use. +- Use `ngx.decode_base64url` instead of `require "ngx.base64".decode_base64url` +- Use go.sha256 instead of unavailable OpenResty `digest.new("sha256")` function. entities/plugins.lua: @@ -86,6 +89,21 @@ tools/sandbox.lua lua-tree/share/lua/5.1/socket/url.lua - override all imports of socket.url by returning patched.url +lua-tree/share/lua/5.1/kong/constants.lua +- remove LOG_LEVELS (depends on ngx) + +lua-tree/share/lua/5.1/kong/pdk/node.lua +- maintain import references but remove package contents + +lua-tree/share/lua/5.1/kong/pdk/private/node.lua +- maintain import references but remove package contents + +lua-tree/share/lua/5.1/kong/router/atc.lua +- maintain import references but remove package contents + +tools/uri.lua +- replace "table.new" with local definition + --- diff --git a/lua-tree/share/lua/5.1/kong/db/schema/entities/certificates.lua b/lua-tree/share/lua/5.1/kong/db/schema/entities/certificates.lua index dbe2d4a..a17dc93 100644 @@ -183,40 +201,31 @@ diff --git a/lua-tree/share/lua/5.1/kong/db/schema/init.lua b/lua-tree/share/lua index 489a666..1c660ed 100644 --- a/lua-tree/share/lua/5.1/kong/db/schema/init.lua +++ b/lua-tree/share/lua/5.1/kong/db/schema/init.lua -@@ -1,10 +1,10 @@ +@@ -1,10 +1,11 @@ local tablex = require "pl.tablex" local pretty = require "pl.pretty" local utils = require "kong.tools.utils" -local cjson = require "cjson" -local new_tab = require "table.new" - local nkeys = require "table.nkeys" +-local nkeys = require "table.nkeys" -local is_reference = require "kong.pdk.vault".new().is_reference ++local nkeys = utils.nkeys ++local new_tab = utils.new_tab + +local cjson = { array_mt = {} } --- TODO(hbagdi) XXX analyze the impact +local is_reference = function(value) return false end local setmetatable = setmetatable -@@ -43,6 +43,18 @@ Schema.__index = Schema +@@ -43,7 +44,6 @@ Schema.__index = Schema local _cache = {} local _workspaceable = {} -+-- `table.new()` is a LuaJIT function and is not available to the Lua VM that goks uses. -+local new_tab -+do -+ local ok -+ ok, new_tab = pcall(require, "table.new") -+ if not ok then -+ new_tab = function(narr, nrec) -+ return {} -+ end -+ end -+end -+ - +- local validation_errors = { -- general message -@@ -315,7 +327,7 @@ Schema.validators = { + ERROR = "Validation error: %s", +@@ -315,7 +315,7 @@ Schema.validators = { if #value ~= 36 then return nil end @@ -225,7 +234,7 @@ index 489a666..1c660ed 100644 end, contains = function(array, wanted) -@@ -956,10 +968,6 @@ function Schema:validate_field(field, value) +@@ -956,10 +956,6 @@ function Schema:validate_field(field, value) field.len_min = 1 end @@ -236,7 +245,7 @@ index 489a666..1c660ed 100644 elseif field.type == "function" then if type(value) ~= "function" then return nil, validation_errors.FUNCTION -@@ -1116,10 +1124,8 @@ validate_fields = function(self, input) +@@ -1116,10 +1112,8 @@ validate_fields = function(self, input) pok, err, errors[k] = pcall(self.validate_field, self, input, v) if not pok then errors[k] = validation_errors.SCHEMA_CANNOT_VALIDATE @@ -247,7 +256,7 @@ index 489a666..1c660ed 100644 else field, err = resolve_field(self, k, field, subschema) if field then -@@ -1208,12 +1214,6 @@ local function run_entity_check(self, name, input, arg, full_check, errors) +@@ -1208,12 +1202,6 @@ local function run_entity_check(self, name, input, arg, full_check, errors) end else all_nil = false @@ -260,7 +269,7 @@ index 489a666..1c660ed 100644 end if errors[fname] then all_ok = false -@@ -1250,7 +1250,7 @@ local function run_entity_check(self, name, input, arg, full_check, errors) +@@ -1250,7 +1238,7 @@ local function run_entity_check(self, name, input, arg, full_check, errors) else local error_fmt = validation_errors[name:upper()] @@ -269,7 +278,7 @@ index 489a666..1c660ed 100644 if not err then local data = pretty.write({ name = arg }):gsub("%s+", " ") err = validation_errors.ENTITY_CHECK:format(name, data) -@@ -1654,14 +1654,7 @@ function Schema:process_auto_fields(data, context, nulls, opts) +@@ -1654,14 +1642,7 @@ function Schema:process_auto_fields(data, context, nulls, opts) -- detected with ngx.ctx.KONG_PHASE, but to limit context -- access we use nulls that admin api sets to true. local kong = kong @@ -285,7 +294,7 @@ index 489a666..1c660ed 100644 local refs local prev_refs = resolve_references and data["$refs"] -@@ -1677,7 +1670,7 @@ function Schema:process_auto_fields(data, context, nulls, opts) +@@ -1677,7 +1658,7 @@ function Schema:process_auto_fields(data, context, nulls, opts) end elseif ftype == "string" then @@ -294,7 +303,7 @@ index 489a666..1c660ed 100644 value = random_string() end -@@ -1722,15 +1715,13 @@ function Schema:process_auto_fields(data, context, nulls, opts) +@@ -1722,15 +1703,13 @@ function Schema:process_auto_fields(data, context, nulls, opts) refs = { [key] = value } end @@ -311,7 +320,7 @@ index 489a666..1c660ed 100644 end value = nil -@@ -1761,15 +1752,13 @@ function Schema:process_auto_fields(data, context, nulls, opts) +@@ -1761,15 +1740,13 @@ function Schema:process_auto_fields(data, context, nulls, opts) refs[key][i] = value[i] @@ -328,6 +337,7 @@ index 489a666..1c660ed 100644 end value[i] = nil + diff --git a/lua-tree/share/lua/5.1/kong/db/schema/plugin_loader.lua b/lua-tree/share/lua/5.1/kong/db/schema/plugin_loader.lua index 5ec62ec..7d0dc4b 100644 --- a/lua-tree/share/lua/5.1/kong/db/schema/plugin_loader.lua @@ -836,7 +846,7 @@ index 08405b8..2b1157d 100644 local get_mime_type local get_error_template do -@@ -1455,19 +1260,9 @@ do +@@ -1455,24 +1260,14 @@ do end end @@ -859,6 +869,96 @@ index 08405b8..2b1157d 100644 local try_decode_base64 do + local decode_base64 = ngx.decode_base64 +- local decode_base64url = require "ngx.base64".decode_base64url ++ local decode_base64url = ngx.decode_base64url + + local function decode_base64_str(str) + if type(str) == "string" then +@@ -1504,42 +1299,15 @@ end + _M.try_decode_base64 = try_decode_base64 + + +-local sha256_bin +-do +- local digest = require "resty.openssl.digest" +- local sha256_digest +- +- function sha256_bin(key) +- local _, bin, err +- if not sha256_digest then +- sha256_digest, err = digest.new("sha256") +- if err then +- return nil, err +- end +- end +- +- bin, err = sha256_digest:final(key) +- if err then +- sha256_digest = nil +- return nil, err +- end +- +- _, err = sha256_digest:reset() +- if err then +- sha256_digest = nil +- end +- +- return bin +- end +-end ++local sha256_bin = require "go.sha256".sha256_bin + _M.sha256_bin = sha256_bin + + + local sha256_hex, sha256_base64, sha256_base64url + do +- local to_hex = require "resty.string".to_hex ++ local to_hex = require "go.hex".to_hex + local to_base64 = ngx.encode_base64 +- local to_base64url = require "ngx.base64".encode_base64url ++ local to_base64url = ngx.encode_base64url + + local function sha256_encode(encode_alg, key) + local bin, err = sha256_bin(key) +@@ -1566,4 +1334,36 @@ _M.sha256_hex = sha256_hex + _M.sha256_base64 = sha256_base64 + _M.sha256_base64url = sha256_base64url + ++-- `table.new()` is a LuaJIT function and is not available to the Lua VM that goks uses. ++local new_tab ++do ++ local ok ++ ok, new_tab = pcall(require, "table.new") ++ if not ok then ++ new_tab = function(narr, nrec) ++ return {} ++ end ++ end ++end ++ ++-- `table.nkeys()` is a LuaJIT function and is not available to the Lua VM that goks uses. ++local nkeys ++do ++ local ok ++ ok, nkeys = pcall(require, "table.nkeys") ++ if not ok then ++ nkeys = function (tab) ++ local count = 0 ++ for _, v in pairs(tab) do ++ if v ~= nil then ++ count = count + 1 ++ end ++ end ++ return count ++ end ++ end ++end ++_M.new_tab = new_tab ++_M.nkeys = nkeys ++ + return _M + diff --git a/lua-tree/share/lua/5.1/patched/url.lua b/lua-tree/share/lua/5.1/patched/url.lua new file mode 100644 index 0000000..360ff39 @@ -1569,3 +1669,1083 @@ index 7809535..b36f576 100644 - -return _M +return require "patched.url" + +diff --git a/lua-tree/share/lua/5.1/kong/constants.lua b/lua-tree/share/lua/5.1/kong/constants.lua +index 775eae0..be9ed67 100644 +--- a/lua-tree/share/lua/5.1/kong/constants.lua ++++ b/lua-tree/share/lua/5.1/kong/constants.lua +@@ -218,25 +218,6 @@ local constants = { + + KEY_FORMATS_MAP = key_formats_map, + KEY_FORMATS = key_formats, +- +- LOG_LEVELS = { +- debug = ngx.DEBUG, +- info = ngx.INFO, +- notice = ngx.NOTICE, +- warn = ngx.WARN, +- error = ngx.ERR, +- crit = ngx.CRIT, +- alert = ngx.ALERT, +- emerg = ngx.EMERG, +- [ngx.DEBUG] = "debug", +- [ngx.INFO] = "info", +- [ngx.NOTICE] = "notice", +- [ngx.WARN] = "warn", +- [ngx.ERR] = "error", +- [ngx.CRIT] = "crit", +- [ngx.ALERT] = "alert", +- [ngx.EMERG] = "emerg", +- }, + } + + for _, v in ipairs(constants.CLUSTERING_SYNC_STATUS) do + +diff --git a/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua b/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua +index 753281a..389d7dc 100644 +--- a/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua ++++ b/lua-tree/share/lua/5.1/kong/db/schema/entities/routes.lua +@@ -1,8 +1,6 @@ + local typedefs = require("kong.db.schema.typedefs") + local deprecation = require("kong.deprecation") + +-local CACHED_SCHEMA = require("kong.router.atc").schema +- + -- ATC router is incompatible with goks + local kong_router_flavor = "traditional" + +@@ -48,13 +46,6 @@ if kong_router_flavor == "expressions" then + { custom_entity_check = { + field_sources = { "expression", "id", }, + fn = function(entity) +- local r = router.new(CACHED_SCHEMA) +- +- local res, err = r:add_matcher(0, entity.id, entity.expression) +- if not res then +- return nil, "Router Expression failed validation: " .. err +- end +- + return true + end, + } }, + +diff --git a/lua-tree/share/lua/5.1/kong/pdk/node.lua b/lua-tree/share/lua/5.1/kong/pdk/node.lua +index 60e6489..85fc2fd 100644 +--- a/lua-tree/share/lua/5.1/kong/pdk/node.lua ++++ b/lua-tree/share/lua/5.1/kong/pdk/node.lua +@@ -1,283 +1 @@ +---- Node-level utilities. +--- +--- @module kong.node +- +-local utils = require "kong.tools.utils" +-local ffi = require "ffi" +-local private_node = require "kong.pdk.private.node" +- +- +-local floor = math.floor +-local lower = string.lower +-local match = string.match +-local gsub = string.gsub +-local sort = table.sort +-local insert = table.insert +-local ngx = ngx +-local shared = ngx.shared +-local C = ffi.C +-local ffi_new = ffi.new +-local ffi_str = ffi.string +- +-local NODE_ID_KEY = "kong:node_id" +- +- +-local node_id +-local shms = {} +-local n_workers = ngx.worker.count() +- +- +-for shm_name, shm in pairs(shared) do +- insert(shms, { +- zone = shm, +- name = shm_name, +- capacity = shm:capacity(), +- }) +-end +- +- +-local function convert_bytes(bytes, unit, scale) +- if not unit or lower(unit) == "b" then +- return floor(bytes) +- end +- +- return utils.bytes_to_str(bytes, unit, scale) +-end +- +- +-local function sort_pid_asc(a, b) +- return a.pid < b.pid +-end +- +- +-local function new(self) +- local _NODE = {} +- +- +- --- +- -- Returns the ID used by this node to describe itself. +- -- +- -- @function kong.node.get_id +- -- @treturn string The v4 UUID used by this node as its ID. +- -- @usage +- -- local id = kong.node.get_id() +- function _NODE.get_id() +- if node_id then +- return node_id +- end +- +- local shm = ngx.shared.kong +- +- local ok, err = shm:safe_add(NODE_ID_KEY, utils.uuid()) +- if not ok and err ~= "exists" then +- error("failed to set 'node_id' in shm: " .. err) +- end +- +- node_id, err = shm:get(NODE_ID_KEY) +- if err then +- error("failed to get 'node_id' in shm: " .. err) +- end +- +- if not node_id then +- error("no 'node_id' set in shm") +- end +- +- return node_id +- end +- +- +- --- +- -- Returns memory usage statistics about this node. +- -- +- -- @function kong.node.get_memory_stats +- -- @tparam[opt] string unit The unit that memory is reported in. Can be +- -- any of `b/B`, `k/K`, `m/M`, or `g/G` for bytes, kibibytes, mebibytes, +- -- or gibibytes, respectively. Defaults to `b` (bytes). +- -- @tparam[opt] number scale The number of digits to the right of the decimal +- -- point. Defaults to 2. +- -- @treturn table A table containing memory usage statistics for this node. +- -- If `unit` is `b/B` (the default), reported values are Lua numbers. +- -- Otherwise, reported values are strings with the unit as a suffix. +- -- @usage +- -- local res = kong.node.get_memory_stats() +- -- -- res will have the following structure: +- -- { +- -- lua_shared_dicts = { +- -- kong = { +- -- allocated_slabs = 12288, +- -- capacity = 24576 +- -- }, +- -- kong_db_cache = { +- -- allocated_slabs = 12288, +- -- capacity = 12288 +- -- } +- -- }, +- -- workers_lua_vms = { +- -- { +- -- http_allocated_gc = 1102, +- -- pid = 18004 +- -- }, +- -- { +- -- http_allocated_gc = 1102, +- -- pid = 18005 +- -- } +- -- } +- -- } +- -- +- -- local res = kong.node.get_memory_stats("k", 1) +- -- -- res will have the following structure: +- -- { +- -- lua_shared_dicts = { +- -- kong = { +- -- allocated_slabs = "12.0 KiB", +- -- capacity = "24.0 KiB", +- -- }, +- -- kong_db_cache = { +- -- allocated_slabs = "12.0 KiB", +- -- capacity = "12.0 KiB", +- -- } +- -- }, +- -- workers_lua_vms = { +- -- { +- -- http_allocated_gc = "1.1 KiB", +- -- pid = 18004 +- -- }, +- -- { +- -- http_allocated_gc = "1.1 KiB", +- -- pid = 18005 +- -- } +- -- } +- -- } +- function _NODE.get_memory_stats(unit, scale) +- -- validate arguments +- +- do +- unit = unit or "b" +- scale = scale or 2 +- +- local pok, perr = pcall(utils.bytes_to_str, 0, unit, scale) +- if not pok then +- error(perr, 2) +- end +- end +- +- local res = { +- workers_lua_vms = self.table.new(n_workers, 0), +- lua_shared_dicts = self.table.new(0, #shms), +- } +- +- -- get workers Lua VM allocated memory +- +- do +- if not shared.kong then +- goto lua_shared_dicts +- end +- +- local keys, err = shared.kong:get_keys() +- if not keys then +- res.workers_lua_vms.err = "could not get kong shm keys: " .. err +- goto lua_shared_dicts +- end +- +- if #keys == 1024 then +- -- Preventive warning log for future Kong developers, in case 'kong' +- -- shm becomes mis-used or over-used. +- ngx.log(ngx.WARN, "ngx.shared.kong:get_keys() returned 1024 keys, ", +- "but it may have more") +- end +- +- for i = 1, #keys do +- local pid = match(keys[i], "kong:mem:(%d+)") +- if not pid then +- goto continue +- end +- +- local w = self.table.new(0, 2) +- w.pid = tonumber(pid) +- +- local count, err = shared.kong:get("kong:mem:" .. pid) +- if err then +- w.err = "could not get worker's HTTP Lua VM memory (pid: " .. +- pid .. "): " .. err +- +- elseif type(count) ~= "number" then +- w.err = "could not get worker's HTTP Lua VM memory (pid: " .. +- pid .. "): reported value is corrupted" +- +- else +- count = count * 1024 -- reported value is in kb +- w.http_allocated_gc = convert_bytes(count, unit, scale) +- end +- +- insert(res.workers_lua_vms, w) +- +- ::continue:: +- end +- +- sort(res.workers_lua_vms, sort_pid_asc) +- end +- +- -- get lua_shared_dicts allocated slabs +- ::lua_shared_dicts:: +- +- for _, shm in ipairs(shms) do +- local allocated = shm.capacity - shm.zone:free_space() +- +- res.lua_shared_dicts[shm.name] = { +- capacity = convert_bytes(shm.capacity, unit, scale), +- allocated_slabs = convert_bytes(allocated, unit, scale), +- } +- end +- +- return res +- end +- +- +- --- +- -- Returns the name used by the local machine. +- -- +- -- @function kong.node.get_hostname +- -- @treturn string The local machine hostname. +- -- @usage +- -- local hostname = kong.node.get_hostname() +- function _NODE.get_hostname() +- local SIZE = 253 -- max number of chars for a hostname +- +- local buf = ffi_new("unsigned char[?]", SIZE) +- local res = C.gethostname(buf, SIZE) +- +- if res == 0 then +- local hostname = ffi_str(buf, SIZE) +- return gsub(hostname, "%z+$", "") +- end +- +- local f = io.popen("/bin/hostname") +- local hostname = f:read("*a") or "" +- f:close() +- return gsub(hostname, "\n$", "") +- end +- +- +- local prefix = self and self.configuration and self.configuration.prefix +- if prefix and self.configuration.role == "data_plane" then +- local id, err = private_node.load_node_id(prefix) +- if id then +- node_id = id +- ngx.log(ngx.DEBUG, "restored node_id from the filesystem: ", node_id) +- +- else +- id = _NODE.get_id() +- if err then +- ngx.log(ngx.WARN, "failed to restore node_id from the filesystem: ", +- err, ", so a new one was generated: ", id) +- end +- end +- end +- +- return _NODE +-end +- +- +-return { +- new = new, +-} ++-- not used in koko at this time +diff --git a/lua-tree/share/lua/5.1/kong/pdk/private/node.lua b/lua-tree/share/lua/5.1/kong/pdk/private/node.lua +index ed299c9..85fc2fd 100644 +--- a/lua-tree/share/lua/5.1/kong/pdk/private/node.lua ++++ b/lua-tree/share/lua/5.1/kong/pdk/private/node.lua +@@ -1,103 +1 @@ +-local log = require "kong.cmd.utils.log" +-local utils = require "kong.tools.utils" +-local pl_file = require "pl.file" +-local pl_path = require "pl.path" +-local pl_dir = require "pl.dir" +- +-local fmt = string.format +- +-local cached_node_id +- +-local function node_id_filename(prefix) +- return pl_path.join(prefix, "kong.id") +-end +- +- +-local function initialize_node_id(prefix) +- if not pl_path.exists(prefix) then +- local ok, err = pl_dir.makepath(prefix) +- if not ok then +- return nil, fmt("failed to create directory %s: %s", prefix, err) +- end +- end +- +- local filename = node_id_filename(prefix) +- +- local file_exists = pl_path.exists(filename) +- +- if file_exists then +- local id, err = pl_file.read(filename) +- if err then +- return nil, fmt("failed to access file %s: %s", filename, err) +- end +- +- if not utils.is_valid_uuid(id) then +- log.debug("file %s contains invalid uuid: %s", filename, id) +- -- set false to override it when it contains an invalid uuid. +- file_exists = false +- end +- end +- +- if not file_exists then +- local id = utils.uuid() +- log.debug("persisting node_id (%s) to %s", id, filename) +- +- local ok, write_err = pl_file.write(filename, id) +- if not ok then +- return nil, fmt("failed to persist node_id to %s: %s", filename, write_err) +- end +- cached_node_id = id +- end +- +- return true +-end +- +- +-local function init_node_id(config) +- if not config then +- return +- end +- +- if not config.prefix or config.role ~= "data_plane" then +- return +- end +- +- local ok, err = initialize_node_id(config.prefix) +- if not ok then +- log.warn(err) +- end +-end +- +- +-local function load_node_id(prefix) +- if not prefix then +- return nil, nil +- end +- +- if cached_node_id then +- return cached_node_id, nil +- end +- +- local filename = node_id_filename(prefix) +- +- if not pl_path.exists(filename) then +- return nil, fmt("file %s does not exist", filename) +- end +- +- local id, read_err = pl_file.read(filename) +- if read_err then +- return nil, fmt("failed to access file %s: %s", filename, read_err) +- end +- +- if not utils.is_valid_uuid(id) then +- return nil, fmt("file %s contains invalid uuid: %q", filename, id) +- end +- +- return id, nil +-end +- +- +-return { +- init_node_id = init_node_id, +- load_node_id = load_node_id, +-} ++-- not used in koko at this time +diff --git a/lua-tree/share/lua/5.1/kong/router/atc.lua b/lua-tree/share/lua/5.1/kong/router/atc.lua +index cd07a00..85fc2fd 100644 +--- a/lua-tree/share/lua/5.1/kong/router/atc.lua ++++ b/lua-tree/share/lua/5.1/kong/router/atc.lua +@@ -1,597 +1 @@ +-local _M = {} +-local _MT = { __index = _M, } +- +- +-local schema = require("resty.router.schema") +-local router = require("resty.router.router") +-local context = require("resty.router.context") +-local lrucache = require("resty.lrucache") +-local server_name = require("ngx.ssl").server_name +-local tb_new = require("table.new") +-local tb_clear = require("table.clear") +-local utils = require("kong.router.utils") +-local yield = require("kong.tools.utils").yield +- +- +-local type = type +-local assert = assert +-local setmetatable = setmetatable +-local pairs = pairs +-local ipairs = ipairs +- +- +-local max = math.max +-local tb_concat = table.concat +-local tb_sort = table.sort +- +- +-local ngx = ngx +-local header = ngx.header +-local var = ngx.var +-local ngx_log = ngx.log +-local get_phase = ngx.get_phase +-local get_method = ngx.req.get_method +-local get_headers = ngx.req.get_headers +-local ngx_WARN = ngx.WARN +-local ngx_ERR = ngx.ERR +- +- +-local sanitize_uri_postfix = utils.sanitize_uri_postfix +-local check_select_params = utils.check_select_params +-local strip_uri_args = utils.strip_uri_args +-local get_service_info = utils.get_service_info +-local add_debug_headers = utils.add_debug_headers +-local get_upstream_uri_v0 = utils.get_upstream_uri_v0 +-local route_match_stat = utils.route_match_stat +- +- +-local MAX_REQ_HEADERS = 100 +-local DEFAULT_MATCH_LRUCACHE_SIZE = utils.DEFAULT_MATCH_LRUCACHE_SIZE +- +- +-local LOGICAL_OR = " || " +-local LOGICAL_AND = " && " +- +- +--- reuse table objects +-local gen_values_t = tb_new(10, 0) +- +- +-local CACHED_SCHEMA +-do +- local str_fields = {"net.protocol", "tls.sni", +- "http.method", "http.host", +- "http.path", "http.raw_path", +- "http.headers.*", +- } +- +- local int_fields = {"net.port", +- } +- +- CACHED_SCHEMA = schema.new() +- +- for _, v in ipairs(str_fields) do +- assert(CACHED_SCHEMA:add_field(v, "String")) +- end +- +- for _, v in ipairs(int_fields) do +- assert(CACHED_SCHEMA:add_field(v, "Int")) +- end +-end +- +- +-local is_empty_field +-do +- local null = ngx.null +- local isempty = require("table.isempty") +- +- is_empty_field = function(f) +- return f == nil or f == null or isempty(f) +- end +-end +- +- +-local function escape_str(str) +- return "\"" .. str:gsub([[\]], [[\\]]):gsub([["]], [[\"]]) .. "\"" +-end +- +- +-local function gen_for_field(name, op, vals, val_transform) +- if is_empty_field(vals) then +- return nil +- end +- +- tb_clear(gen_values_t) +- +- local values_n = 0 +- local values = gen_values_t +- +- for _, p in ipairs(vals) do +- values_n = values_n + 1 +- local op = (type(op) == "string") and op or op(p) +- values[values_n] = name .. " " .. op .. " " .. +- escape_str(val_transform and val_transform(op, p) or p) +- end +- +- if values_n > 0 then +- return "(" .. tb_concat(values, LOGICAL_OR) .. ")" +- end +- +- return nil +-end +- +- +-local function add_atc_matcher(inst, route, route_id, +- get_exp_and_priority, +- remove_existing) +- +- local exp, priority = get_exp_and_priority(route) +- +- if not exp then +- return nil, "could not find expression, route: " .. route_id +- end +- +- if remove_existing then +- assert(inst:remove_matcher(route_id)) +- end +- +- local ok, err = inst:add_matcher(priority, route_id, exp) +- if not ok then +- return nil, "could not add route: " .. route_id .. ", err: " .. err +- end +- +- return true +-end +- +- +-local function is_http_headers_field(field) +- return field:sub(1, 13) == "http.headers." +-end +- +- +-local function has_header_matching_field(fields) +- for _, field in ipairs(fields) do +- if is_http_headers_field(field) then +- return true +- end +- end +- +- return false +-end +- +- +-local function new_from_scratch(routes, get_exp_and_priority) +- local phase = get_phase() +- local inst = router.new(CACHED_SCHEMA) +- +- local routes_n = #routes +- local routes_t = tb_new(0, routes_n) +- local services_t = tb_new(0, routes_n) +- +- local new_updated_at = 0 +- +- for _, r in ipairs(routes) do +- local route = r.route +- local route_id = route.id +- +- if not route_id then +- return nil, "could not categorize route" +- end +- +- routes_t[route_id] = route +- services_t[route_id] = r.service +- +- local ok, err = add_atc_matcher(inst, route, route_id, +- get_exp_and_priority, false) +- if ok then +- new_updated_at = max(new_updated_at, route.updated_at or 0) +- +- else +- ngx_log(ngx_ERR, err) +- +- routes_t[route_id] = nil +- services_t[route_id] = nil +- end +- +- yield(true, phase) +- end +- +- local fields = inst:get_fields() +- local match_headers = has_header_matching_field(fields) +- +- return setmetatable({ +- schema = CACHED_SCHEMA, +- router = inst, +- routes = routes_t, +- services = services_t, +- fields = fields, +- match_headers = match_headers, +- updated_at = new_updated_at, +- rebuilding = false, +- }, _MT) +-end +- +- +-local function new_from_previous(routes, get_exp_and_priority, old_router) +- if old_router.rebuilding then +- return nil, "concurrent incremental router rebuild without mutex, this is unsafe" +- end +- +- old_router.rebuilding = true +- +- local phase = get_phase() +- +- local inst = old_router.router +- local old_routes = old_router.routes +- local old_services = old_router.services +- +- local updated_at = old_router.updated_at +- local new_updated_at = 0 +- +- -- create or update routes +- for _, r in ipairs(routes) do +- local route = r.route +- local route_id = route.id +- +- if not route_id then +- return nil, "could not categorize route" +- end +- +- local old_route = old_routes[route_id] +- local route_updated_at = route.updated_at +- +- route.seen = true +- +- old_routes[route_id] = route +- old_services[route_id] = r.service +- +- local ok = true +- local err +- +- if not old_route then +- -- route is new +- ok, err = add_atc_matcher(inst, route, route_id, get_exp_and_priority, false) +- +- elseif route_updated_at >= updated_at or +- route_updated_at ~= old_route.updated_at then +- +- -- route is modified (within a sec) +- ok, err = add_atc_matcher(inst, route, route_id, get_exp_and_priority, true) +- end +- +- if ok then +- new_updated_at = max(new_updated_at, route_updated_at) +- +- else +- ngx_log(ngx_ERR, err) +- +- old_routes[route_id] = nil +- old_services[route_id] = nil +- end +- +- yield(true, phase) +- end +- +- -- remove routes +- for id, r in pairs(old_routes) do +- if r.seen then +- r.seen = nil +- +- else +- assert(inst:remove_matcher(id)) +- +- old_routes[id] = nil +- old_services[id] = nil +- end +- +- yield(true, phase) +- end +- +- local fields = inst:get_fields() +- +- old_router.fields = fields +- old_router.match_headers = has_header_matching_field(fields) +- old_router.updated_at = new_updated_at +- old_router.rebuilding = false +- +- return old_router +-end +- +- +-function _M.new(routes, cache, cache_neg, old_router, get_exp_and_priority) +- if type(routes) ~= "table" then +- return error("expected arg #1 routes to be a table") +- end +- +- local router, err +- +- if not old_router then +- router, err = new_from_scratch(routes, get_exp_and_priority) +- +- else +- router, err = new_from_previous(routes, get_exp_and_priority, old_router) +- end +- +- if not router then +- return nil, err +- end +- +- router.cache = cache or lrucache.new(DEFAULT_MATCH_LRUCACHE_SIZE) +- router.cache_neg = cache_neg or lrucache.new(DEFAULT_MATCH_LRUCACHE_SIZE) +- +- return router +-end +- +- +--- split port in host, ignore form '[...]' +--- example.com:123 => example.com, 123 +--- example.*:123 => example.*, 123 +-local split_host_port +-do +- local tonumber = tonumber +- local DEFAULT_HOSTS_LRUCACHE_SIZE = DEFAULT_MATCH_LRUCACHE_SIZE +- +- local memo_hp = lrucache.new(DEFAULT_HOSTS_LRUCACHE_SIZE) +- +- split_host_port = function(key) +- if not key then +- return nil, nil +- end +- +- local m = memo_hp:get(key) +- +- if m then +- return m[1], m[2] +- end +- +- local p = key:find(":", nil, true) +- if not p then +- memo_hp:set(key, { key, nil }) +- return key, nil +- end +- +- local port = tonumber(key:sub(p + 1)) +- +- if not port then +- memo_hp:set(key, { key, nil }) +- return key, nil +- end +- +- local host = key:sub(1, p - 1) +- +- memo_hp:set(key, { host, port }) +- +- return host, port +- end +-end +- +- +-function _M:select(req_method, req_uri, req_host, req_scheme, +- src_ip, src_port, +- dst_ip, dst_port, +- sni, req_headers) +- +- check_select_params(req_method, req_uri, req_host, req_scheme, +- src_ip, src_port, +- dst_ip, dst_port, +- sni, req_headers) +- +- local c = context.new(self.schema) +- +- local host, port = split_host_port(req_host) +- +- for _, field in ipairs(self.fields) do +- if field == "http.method" then +- assert(c:add_value("http.method", req_method)) +- +- elseif field == "http.path" then +- assert(c:add_value("http.path", req_uri)) +- +- elseif field == "http.host" then +- assert(c:add_value("http.host", host)) +- +- elseif field == "net.port" then +- assert(c:add_value("net.port", port)) +- +- elseif field == "net.protocol" then +- assert(c:add_value("net.protocol", req_scheme)) +- +- elseif field == "tls.sni" then +- assert(c:add_value("tls.sni", sni)) +- +- elseif req_headers and is_http_headers_field(field) then +- local h = field:sub(14) +- local v = req_headers[h] +- +- if v then +- if type(v) == "string" then +- assert(c:add_value(field, v:lower())) +- +- else +- for _, v in ipairs(v) do +- assert(c:add_value(field, v:lower())) +- end +- end +- end +- end +- end +- +- local matched = self.router:execute(c) +- if not matched then +- return nil +- end +- +- local uuid, matched_path, captures = c:get_result("http.path") +- +- local service = self.services[uuid] +- local matched_route = self.routes[uuid] +- +- local service_protocol, _, --service_type +- service_host, service_port, +- service_hostname_type, service_path = get_service_info(service) +- +- local request_prefix = matched_route.strip_path and matched_path or nil +- local request_postfix = request_prefix and req_uri:sub(#matched_path + 1) or req_uri:sub(2, -1) +- request_postfix = sanitize_uri_postfix(request_postfix) or "" +- local upstream_base = service_path or "/" +- +- local upstream_uri = get_upstream_uri_v0(matched_route, request_postfix, req_uri, +- upstream_base) +- +- return { +- route = matched_route, +- service = service, +- prefix = request_prefix, +- matches = { +- uri_captures = (captures and captures[1]) and captures or nil, +- }, +- upstream_url_t = { +- type = service_hostname_type, +- host = service_host, +- port = service_port, +- }, +- upstream_scheme = service_protocol, +- upstream_uri = upstream_uri, +- upstream_host = matched_route.preserve_host and req_host or nil, +- } +-end +- +- +-local get_headers_key +-do +- local headers_t = tb_new(8, 0) +- +- get_headers_key = function(headers) +- tb_clear(headers_t) +- +- local headers_count = 0 +- +- for name, value in pairs(headers) do +- local name = name:gsub("-", "_"):lower() +- +- if type(value) == "table" then +- for i, v in ipairs(value) do +- value[i] = v:lower() +- end +- tb_sort(value) +- value = tb_concat(value, ", ") +- +- else +- value = value:lower() +- end +- +- headers_t[headers_count + 1] = "|" +- headers_t[headers_count + 2] = name +- headers_t[headers_count + 3] = "=" +- headers_t[headers_count + 4] = value +- +- headers_count = headers_count + 4 +- end +- +- return tb_concat(headers_t, nil, 1, headers_count) +- end +-end +- +- +-function _M:exec(ctx) +- local req_method = get_method() +- local req_uri = ctx and ctx.request_uri or var.request_uri +- local req_host = var.http_host +- local sni = server_name() +- +- local headers, headers_key +- if self.match_headers then +- local err +- headers, err = get_headers(MAX_REQ_HEADERS) +- if err == "truncated" then +- ngx_log(ngx_WARN, "retrieved ", MAX_REQ_HEADERS, " headers for evaluation ", +- "(max) but request had more; other headers will be ignored") +- end +- +- headers["host"] = nil +- +- headers_key = get_headers_key(headers) +- end +- +- req_uri = strip_uri_args(req_uri) +- +- -- cache lookup +- +- local cache_key = (req_method or "") .. "|" .. +- (req_uri or "") .. "|" .. +- (req_host or "") .. "|" .. +- (sni or "") .. (headers_key or "") +- +- local match_t = self.cache:get(cache_key) +- if not match_t then +- if self.cache_neg:get(cache_key) then +- route_match_stat(ctx, "neg") +- return nil +- end +- +- local req_scheme = ctx and ctx.scheme or var.scheme +- +- match_t = self:select(req_method, req_uri, req_host, req_scheme, +- nil, nil, nil, nil, +- sni, headers) +- if not match_t then +- self.cache_neg:set(cache_key, true) +- return nil +- end +- +- self.cache:set(cache_key, match_t) +- +- else +- route_match_stat(ctx, "pos") +- end +- +- -- found a match +- +- -- debug HTTP request header logic +- add_debug_headers(var, header, match_t) +- +- return match_t +-end +- +- +-function _M._set_ngx(mock_ngx) +- if type(mock_ngx) ~= "table" then +- return +- end +- +- if mock_ngx.header then +- header = mock_ngx.header +- end +- +- if mock_ngx.var then +- var = mock_ngx.var +- end +- +- if mock_ngx.log then +- ngx_log = mock_ngx.log +- end +- +- if type(mock_ngx.req) == "table" then +- if mock_ngx.req.get_method then +- get_method = mock_ngx.req.get_method +- end +- +- if mock_ngx.req.get_headers then +- get_headers = mock_ngx.req.get_headers +- end +- end +-end +- +- +-_M.schema = CACHED_SCHEMA +- +-_M.LOGICAL_OR = LOGICAL_OR +-_M.LOGICAL_AND = LOGICAL_AND +- +-_M.escape_str = escape_str +-_M.is_empty_field = is_empty_field +-_M.gen_for_field = gen_for_field +-_M.split_host_port = split_host_port +- +- +-return _M ++-- not used in koko at this time + +diff --git a/lua-tree/share/lua/5.1/kong/tools/uri.lua b/lua-tree/share/lua/5.1/kong/tools/uri.lua +index ecc5991..e8ae945 100644 +--- a/lua-tree/share/lua/5.1/kong/tools/uri.lua ++++ b/lua-tree/share/lua/5.1/kong/tools/uri.lua +@@ -1,4 +1,4 @@ +-local table_new = require "table.new" ++local table_new = require "kong.tools.utils".new_tab + + local string_char = string.char + local string_upper = string.upper +@@ -40,7 +40,7 @@ end + + local ESCAPE_PATTERN = "[^!#$&'()*+,/:;=?@[\\]A-Z\\d-_.~%]" + +-local TMP_OUTPUT = require("table.new")(16, 0) ++local TMP_OUTPUT = table_new(16, 0) + local DOT = string_byte(".") + local SLASH = string_byte("/")