From a468bc573d094612e8d9d5b8adb97dd7ada3ccc1 Mon Sep 17 00:00:00 2001
From: Yochem van Rosmalen <git@yochem.nl>
Date: Tue, 18 Nov 2025 21:35:22 +0100
Subject: ci: check URL reachability #35593

Problem:
scripts/check_urls.vim manually matches urls in the help pages and then
synchronously checks them via curl/wget/powershell. This is extremely
slow (~5 minutes for Nvims runtime on my machine) and prone to errors in
how the urls are matched.

Solution:
- Use Tree-sitter to find the urls in the help pages and `vim.net.request` to
  check the responses.
- Add a `lintdocurls` build task and check it in CI (every Friday).
  - Reopens a dedicated issue if it finds unreachable URLs.
- Drop the old check_urls.vim script.
---
 scripts/check_urls.vim | 89 --------------------------------------------------
 scripts/lintdoc.lua    |  2 +-
 scripts/vim-patch.sh   |  6 +---
 3 files changed, 2 insertions(+), 95 deletions(-)
 delete mode 100644 scripts/check_urls.vim

(limited to 'scripts')

diff --git a/scripts/check_urls.vim b/scripts/check_urls.vim
deleted file mode 100644
index b75dc29c48..0000000000
--- a/scripts/check_urls.vim
+++ /dev/null
@@ -1,89 +0,0 @@
-" Test for URLs in help documents.
-"
-" Opens a new window with all found URLS followed by return code from curl
-" (anything other than 0 means unreachable)
-"
-" Written by Christian Brabandt.
-
-func Test_check_URLs()
-"20.10.23, added by Restorer
-  if has("win32")
-    let s:outdev = 'nul'
-  else
-    let s:outdev = '/dev/null'
-  endif
-" Restorer: For Windows users. If "curl" or "wget" is installed on the system
-" but not in %PATH%, add the full path to them to %PATH% environment variable.
-  if executable('curl')
-    " Note: does not follow redirects!
-    let s:command1 = 'curl --silent --max-time 5 --fail --output ' ..s:outdev.. ' --head '
-    let s:command2 = ""
-  elseif executable('wget')
-    " Note: only allow a couple of redirects
-    let s:command1 = 'wget --quiet -S --spider --max-redirect=2 --timeout=5 --tries=2 -O ' ..s:outdev.. ' '
-    let s:command2 = ""
-  elseif has("win32") "20.10.23, added by Restorer
-    if executable('powershell')
-      if 2 == system('powershell -nologo -noprofile "$psversiontable.psversion.major"')
-        echoerr 'To work in OS Windows requires the program "PowerShell" version 3.0 or higher'
-        return
-      endif
-      let s:command1 = 
-            \ "powershell -nologo -noprofile \"{[Net.ServicePointManager]::SecurityProtocol = 'Tls12, Tls11, Tls, Ssl3'};try{(Invoke-WebRequest -MaximumRedirection 2 -TimeoutSec 5 -Uri "
-      let s:command2 = ').StatusCode}catch{exit [int]$Error[0].Exception.Status}"'
-    endif
-  else
-    echoerr 'Only works when "curl" or "wget", or "powershell" is available'
-    return
-  endif
-
-  " Do the testing.
-  set report =999
-  set nomore shm +=s
-
-  let pat='\(https\?\|ftp\)://[^\t* ]\+'
-  exe 'helpgrep' pat
-  helpclose
-
-  let urls = map(getqflist(), 'v:val.text')
-  " do not use submatch(1)!
-  let urls = map(urls, {key, val -> matchstr(val, pat)})
-  " remove examples like user@host (invalid urls)
-  let urls = filter(urls, 'v:val !~ "@"')
-  " Remove example URLs which are invalid
-  let urls = filter(urls, {key, val -> val !~ '\<\(\(my\|some\)\?host\|machine\|hostname\|file\)\>'})
-  new
-  put =urls
-  " remove some more invalid items
-  " empty lines
-  "20.10.23, Restorer: '_' is a little faster, see `:h global`
-  v/./d _
-  " remove # anchors
-  %s/#.*$//e
-  " remove trailing stuff (parenthesis, dot, comma, quotes), but only for HTTP
-  " links
-  g/^h/s#[.),'"`/>][:.,]\?$##
-  g#^[hf]t\?tp:/\(/\?\.*\)$#d _
-  silent! g/ftp://,$/d _
-  silent! g/=$/d _
-  let a = getline(1,'$')
-  let a = uniq(sort(a))
-  %d _
-  call setline(1, a)
-
-  %s/.*/\=TestURL(submatch(0))/
-
-  " highlight the failures
-  /.* \([0-9]*[1-9]\|[0-9]\{2,}\)$
-endfunc
-
-func TestURL(url)
-  " Relies on the return code to determine whether a page is valid
-  echom printf("Testing URL: %d/%d %s", line('.'), line('$'), a:url)
-  call system(s:command1 .. shellescape(a:url) .. s:command2)
-  return printf("%s %d", a:url, v:shell_error)
-endfunc
-
-call Test_check_URLs()
-
-" vim: sw=2 sts=2 et
diff --git a/scripts/lintdoc.lua b/scripts/lintdoc.lua
index d246c51176..a81dbd0d74 100755
--- a/scripts/lintdoc.lua
+++ b/scripts/lintdoc.lua
@@ -16,7 +16,7 @@ print('Running lintdoc ...')
 vim.cmd [[ packadd netrw ]]
 vim.cmd [[ helptags ALL ]]
 
-require('src.gen.gen_help_html').run_validate()
+require('src.gen.gen_help_html').run_validate(nil, _G.arg[1] ~= nil)
 require('src.gen.gen_help_html').test_gen()
 
 print('lintdoc PASSED.')
diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh
index eda530e10a..2cef123c93 100755
--- a/scripts/vim-patch.sh
+++ b/scripts/vim-patch.sh
@@ -208,7 +208,7 @@ preprocess_patch() {
   2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git [ab]/runtime/\<\%('"${na_rt}"'\)\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
   # Remove unwanted Vim doc files.
-  local na_doc='channel\.txt\|if_cscop\.txt\|netbeans\.txt\|os_\w\+\.txt\|print\.txt\|term\.txt\|testing\.txt\|todo\.txt\|vim9\.txt\|tags'
+  local na_doc='channel\.txt\|if_cscop\.txt\|netbeans\.txt\|os_\w\+\.txt\|print\.txt\|term\.txt\|testing\.txt\|todo\.txt\|vim9\.txt\|tags\|test_urls\.vim'
   2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git [ab]/runtime/doc/\<\%('"${na_doc}"'\)\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
   # Remove "Last change ..." changes in doc files.
@@ -327,10 +327,6 @@ preprocess_patch() {
   LC_ALL=C sed -Ee 's/( [ab]\/runtime\/doc)\/sponsor\.txt/\1\/intro.txt/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
-  # Rename test_urls.vim to check_urls.vim
-  LC_ALL=C sed -Ee 's/( [ab])\/runtime\/doc\/test(_urls\.vim)/\1\/scripts\/check\2/g' \
-    "$file" > "$file".tmp && mv "$file".tmp "$file"
-
   # Rename path to check_colors.vim
   LC_ALL=C sed -Ee 's/( [ab]\/runtime)\/colors\/(tools\/check_colors\.vim)/\1\/\2/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
-- 
cgit v1.3-3-g829e