backport: fix(api): `LuaRef` leak in `nvim_set_keymap` on LHS too long (>=66 bytes) (#39376)

Problem: `nvim_set_keymap` leaks the `callback` `LuaRef` when the
LHS is too long.

Solution: Make `set_maparg_lhs_rhs` transfer `rhs_lua` to
`MapArguments` up front so the caller always owns the ref.

(cherry picked from commit 58aad59e1cf89e2bee0fc2e02c42506d2b1feeaf)

Co-authored-by: Barrett Ruth <62671086+barrettruth@users.noreply.github.com>

2 files changed, 18 insertions, 0 deletions

diff --git a/src/nvim/mapping.c b/src/nvim/mapping.c
index 3d823f6f51..c9b8763354 100644
--- a/src/nvim/mapping.c
+++ b/src/nvim/mapping.c
@@ -302,6 +302,7 @@ static bool set_maparg_lhs_rhs(const char *const orig_lhs, const size_t orig_lhs
                                const LuaRef rhs_lua, const char *const cpo_val,
                                MapArguments *const mapargs)
 {
+  mapargs->rhs_lua = rhs_lua;
   char lhs_buf[128];
 
   // If mapping has been given as ^V<C_UP> say, then replace the term codes
diff --git a/test/functional/api/keymap_spec.lua b/test/functional/api/keymap_spec.lua
index 9b7cceb266..fe28739c97 100644
--- a/test/functional/api/keymap_spec.lua
+++ b/test/functional/api/keymap_spec.lua
@@ -617,6 +617,23 @@ describe('nvim_set_keymap, nvim_del_keymap', function()
     eq('LHS exceeds maximum map length: ' .. lhs, pcall_err(api.nvim_del_keymap, '', lhs))
   end)
 
+  it('does not leak callback LuaRef on too-long LHS #39351', function()
+    eq(
+      0,
+      exec_lua(function()
+        local weak = setmetatable({}, { __mode = 'v' })
+        for i = 1, 2 do
+          local cb = function() end
+          weak[i] = cb
+          local ok = pcall(vim.api.nvim_set_keymap, 'n', ('a'):rep(66), '', { callback = cb })
+          assert(not ok)
+        end
+        collectgarbage('collect')
+        return vim.tbl_count(weak)
+      end)
+    )
+  end)
+
   it('does not throw errors when rhs is longer than MAXMAPLEN', function()
     local MAXMAPLEN = 50
     local rhs = ''