Title: A simple script to benchmark/exercise vim syntax highlighting
Date: 2026-10-12 15:00

Because I [dislike tree-sitter]({filename}/dev/disabling_treesitter_in_neovim.md ),
I've been looking at optimizing vim syntax-based code highlighting, sending
[~30 pull
requests](https://github.com/vim/vim/pulls?q=is%3Apr+state%3Aclosed+author%3Ajvoisin+created%3A2026)
as a result. It's surprisingly non-trivial to profile syntax highlighting, as
vim has some [pty](https://en.wikipedia.org/wiki/Pseudoterminal)/terminal
relate checks, as well as a bunch of screen/display optimizations, so I've been
using the following script quite extensively:

```vimscript
" Profile a runtime syntax file with a sequential full-buffer parse.
" Pass this as Vim's vimrc, e.g.:
"   src/vim -Nu profile_syntax.vim file.py
" Inputs are extended to 100,000 lines. The report is written to
" output-YYYYMMDD-HHMMSS.txt in the current directory.
set nocompatible
execute 'set runtimepath^=' . fnameescape(getcwd() . '/runtime')
filetype on

function! ProfileSyntax()
  let input_lines = getline(1, '$')
  let target_lines = 100000
  " Repeat the original input, using a partial final copy if needed.
  while line('$') < target_lines
    let copy_count = min([len(input_lines), target_lines - line('$')])
    call append(line('$'), input_lines[: copy_count - 1])
  endwhile

  let syntax_name = &l:filetype
  if empty(syntax_name)
    echoerr 'No filetype detected; cannot profile syntax.'
    qall!
  endif

  syntax clear
  unlet! b:current_syntax
  " Load through 'runtimepath' so compound filetypes, after/syntax and
  " :runtime! includes all resolve the way a normal session would.
  execute 'runtime! syntax/' . syntax_name . '.vim'
  if empty(get(b:, 'current_syntax', ''))
    echoerr 'Cannot load syntax for: ' . syntax_name
    qall!
  endif
  syntime on

  for lnum in range(1, line('$'))
    call synID(lnum, 1, 1)
  endfor

  let report_file = getcwd() .. '/output-' .. strftime('%Y%m%d-%H%M%S') .. '.txt'
  let saved_columns = &columns
  let &columns = max([saved_columns, 2000])
  let raw_report = execute('syntime report')
  let &columns = saved_columns

  let entries = []
  let total = []
  let name_width = strlen('NAME')
  for report_line in split(raw_report, "\n")
    if report_line =~# '^\s*TOTAL\s\+COUNT\>'
      continue
    endif
    let fields = matchlist(report_line,
          \ '^\s*\(\S\+\)\s\+\(\S\+\)\s\+\(\S\+\)\s\+\(\S\+\)\s\+\(\S\+\)\s\+\(\S\+\)\s\+\(.*\)$')
    if !empty(fields)
      call add(entries, fields[1:7])
      let name_width = max([name_width, strlen(fields[6])])
    else
      let fields = matchlist(report_line, '^\s*\(\S\+\)\s\+\(\d\+\)\s*$')
      if !empty(fields)
        let total = fields[1:2]
      endif
    endif
  endfor

    let aligned_report = [printf('Syntax: %s  Lines: %d', syntax_name, line('$')), '']
    call add(aligned_report, printf('%-13s %8s %7s %12s %12s %-*s %s',
        \ 'TOTAL', 'COUNT', 'MATCH', 'SLOWEST', 'AVERAGE', name_width,
      \ 'NAME', 'PATTERN'))
  for fields in entries
    call add(aligned_report, printf('%-13s %8s %7s %12s %12s %-*s %s',
          \ fields[0], fields[1], fields[2], fields[3], fields[4],
          \ name_width, fields[5], fields[6]))
  endfor
  if !empty(total)
    call add(aligned_report, '')
    call add(aligned_report, printf('TOTAL TIME: %s    TOTAL COUNT: %s',
          \ total[0], total[1]))
  endif
  call writefile(aligned_report, report_file)
  qall!
endfunction

autocmd VimEnter * call ProfileSyntax()
```

Running it on a source code file, like
[`src/vim9script.c`](https://github.com/vim/vim/blob/master/src/vim9script.c)
with `vim -Nu profile_syntax.vim src/vim9script.c` produces an
`output-$TIMESTAMP.txt` file looking like this:

```
Syntax: c  Lines: 100000

TOTAL            COUNT   MATCH      SLOWEST      AVERAGE NAME               PATTERN
0.102609705      99328    9312  0.000011000  0.000001033 cString            \%(L\|U\|u8\)\="
0.092528889      90839    4198  0.000023291  0.000001019 cCppString         \%(L\|U\|u8\)\="
0.069483028      81272    6177  0.000032958  0.000000855 cNumbers           \<\d\|\.\d
0.062877236      91583       0  0.000010500  0.000000687 cString            \%(U\|u8\=\)"
0.057239944     102135   10552  0.000009083  0.000000560 cBracket           \[\|<::\@!
0.054717217      96789    5371  0.000015541  0.000000565 cCharacter         L\='[^\\]'
0.053602548      91583       0  0.000012208  0.000000585 cSpecialCharacter  L\='\\['\"?\\abfnrtv]'
```

From there, it's simply a matter of tweaking the regex trying make the numbers
go down. Classic tricks include [transforming things like `A[BC]D`
into](https://github.com/vim/vim/pull/21513/changes/dca126c1da4c828828cd89468e9bb958d10b7e7f)
[`ABD` and
`ACD`](https://github.com/vim/vim/pull/21513/changes/74ba41f17e5736252dc8cc0ef7bc1111e9ea3233),
[grouping common patterns
together](https://github.com/vim/vim/pull/21195/changes), [forcing
usage](https://github.com/vim/vim/pull/21513/changes/e5ed2dd06132503339f28067fd214c846bd398f4)
[of the
old](https://github.com/vim/vim/pull/21513/changes/95d2f78939450e8b2c9fd91e897bb37cb9271a6b)
[regex engine](https://github.com/vim/vim/pull/21194), [using plain text
instead of regex](https://github.com/vim/vim/pull/21155/changes), …

The script was also used to optimize the regex engines themselves, by running
it in `perf record` via `perf record -g --call-graph dwarf -- src/vim -Nu
profile_syntax.vim --noplugin -N -X -n -i NONE --not-a-term a_big_file.c` with
its reporting part removed, calling `perf script -i perf.data >
perf.txt` on the results, and dropping the resulting `perf.txt` file in
[speedscope](https://www.speedscope.app/). Tentative improvements were then
verified by running `perf stat -e instructions -- src/vim -Nu
profile_syntax.vim --noplugin -N -X -n -i NONE --not-a-term a_big_file.c` to
see if the number of instructions went down, and validated in speedscope.