alex/neovim
1
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2026-01-06 11:27:26 +10:00
Code Issues Packages Projects Releases Wiki Activity

refactor(build): split out metadata step

Browse Source 
This was part of an attempt to add "git describe" to zig build
without re-building too much. This doesn't yet work as zig upstream
changes are are also needed, but I think this was a sensible refactor
even in isolation, so breaking it out.

"API dispatch" doesn't depend on any ui event stuff nor on
version, that was just an accident of the metadata collection
being crammed into the same file.

Remove "manual invocation" instruction. this is going to bitrot anyway.
Both cmake and build.zig allows you to extract the command line of a
step, so you can debug it separately.
This commit is contained in:
bfredl
2025-10-28 11:43:30 +01:00
parent 9875a9b691
commit 829a5dc83e
4 changed files with 141 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
src/gen/gen_api_dispatch.lua
View File

@@ -1,30 +1,21 @@
-- Generates C code to bridge API <=> Lua.
--
-- Example (manual) invocation:
--
-- make
-- cp build/nvim_version.lua src/nvim/
-- cd src/nvim
-- nvim -l generators/gen_api_dispatch.lua "../../build/src/nvim/auto/api/private/dispatch_wrappers.generated.h" "../../build/src/nvim/auto/api/private/api_metadata.generated.h" "../../build/funcs_metadata.mpack" "../../build/src/nvim/auto/lua_api_c_bindings.generated.h" "../../build/src/nvim/auto/keysets_defs.generated.h" "../../build/ui_metadata.mpack" "../../build/cmake.config/auto/versiondef_git.h" "./api/autocmd.h" "./api/buffer.h" "./api/command.h" "./api/deprecated.h" "./api/extmark.h" "./api/keysets_defs.h" "./api/options.h" "./api/tabpage.h" "./api/ui.h" "./api/vim.h" "./api/vimscript.h" "./api/win_config.h" "./api/window.h" "../../build/include/api/autocmd.h.generated.h" "../../build/include/api/buffer.h.generated.h" "../../build/include/api/command.h.generated.h" "../../build/include/api/deprecated.h.generated.h" "../../build/include/api/extmark.h.generated.h" "../../build/include/api/options.h.generated.h" "../../build/include/api/tabpage.h.generated.h" "../../build/include/api/ui.h.generated.h" "../../build/include/api/vim.h.generated.h" "../../build/include/api/vimscript.h.generated.h" "../../build/include/api/win_config.h.generated.h" "../../build/include/api/window.h.generated.h"
-- to obtain how the script is invoked, look in build/build.ninja and grep for
-- "gen_api_dispatch.lua"
local hashy = require 'gen.hashy'
local c_grammar = require('gen.c_grammar')
-- output h file with generated dispatch functions (dispatch_wrappers.generated.h)
local dispatch_outputf = arg[1]
-- output h file with packed metadata (api_metadata.generated.h)
local api_metadata_outputf = arg[2]
-- output metadata mpack file, for use by other build scripts (funcs_metadata.mpack)
local mpack_outputf = arg[3]
-- output file with exported functions metadata
local exported_funcs_metadata_outputf = arg[2]
-- output mpack file with raw metadata, for use by gen_eval.lua (funcs_metadata.mpack)
local eval_funcs_metadata_outputf = arg[3]
local lua_c_bindings_outputf = arg[4] -- lua_api_c_bindings.generated.c
local keysets_outputf = arg[5] -- keysets_defs.generated.h
local ui_metadata_inputf = arg[6] -- ui events metadata
local git_version_inputf = arg[7] -- git version header
local nvim_version_inputf = arg[8] -- nvim version
local dump_bin_array_inputf = arg[9]
local dispatch_deprecated_inputf = arg[10]
local pre_args = 10
assert(#arg >= pre_args)
local dispatch_deprecated_inputf = arg[6]
local pre_args = 6
assert(#arg >= 6)
local function real_type(type, exported)
local ptype = c_grammar.typed_container:match(type)
@@ -165,8 +156,6 @@ local function add_keyset(val)
}
end
local ui_options_text = nil
-- read each input file, parse and append to the api metadata
for i = pre_args + 1, #arg do
local full_path = arg[i]
@@ -190,12 +179,9 @@ for i = pre_args + 1, #arg do
end
end
ui_options_text = ui_options_text or text:match('ui_ext_names%[][^{]+{([^}]+)}')
input:close()
end
--- @cast ui_options_text string
--- @generic T: table
--- @param orig T
--- @return T
@@ -303,77 +289,11 @@ for _, f in ipairs(functions) do
end
end
local ui_options = { 'rgb' }
for x in ui_options_text:gmatch('"([a-z][a-z_]+)"') do
table.insert(ui_options, x)
end
local metadata_output = assert(io.open(exported_funcs_metadata_outputf, 'wb'))
metadata_output:write(vim.mpack.encode(exported_functions))
metadata_output:close()
--- @type integer[]
local version = loadfile(nvim_version_inputf)()
local git_version = io.open(git_version_inputf):read '*a'
local version_build = git_version:match('#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL
local pieces = {} --- @type string[]
-- Naively using mpack.encode({foo=x, bar=y}) will make the build
-- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead
local function fixdict(num)
if num > 15 then
error 'implement more dict codes'
end
pieces[#pieces + 1] = string.char(128 + num)
end
local function put(item, item2)
table.insert(pieces, vim.mpack.encode(item))
if item2 ~= nil then
table.insert(pieces, vim.mpack.encode(item2))
end
end
fixdict(6)
put('version')
fixdict(1 + #version)
for _, item in ipairs(version) do
-- NB: all items are mandatory. But any error will be less confusing
-- with placeholder vim.NIL (than invalid mpack data)
local val = item[2] == nil and vim.NIL or item[2]
put(item[1], val)
end
put('build', version_build)
put('functions', exported_functions)
put('ui_events')
table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all'))
put('ui_options', ui_options)
put('error_types')
fixdict(2)
put('Exception', { id = 0 })
put('Validation', { id = 1 })
put('types')
local types =
{ { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } }
fixdict(#types)
for i, item in ipairs(types) do
put(item[1])
fixdict(2)
put('id', i - 1)
put('prefix', item[2])
end
local packed = table.concat(pieces)
--- @type fun(api_metadata: file*, name: string, packed: string)
local dump_bin_array = loadfile(dump_bin_array_inputf)()
-- serialize the API metadata using msgpack and embed into the resulting
-- binary for easy querying by clients
local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb'))
dump_bin_array(api_metadata_output, 'packed_api_metadata', packed)
api_metadata_output:close()
-- start building the dispatch wrapper output
local output = assert(io.open(dispatch_outputf, 'wb'))
@@ -757,7 +677,7 @@ output:close()
--- @cast functions {[integer]: gen_api_dispatch.Function, keysets: gen_api_dispatch.Keyset[]}
functions.keysets = keysets
local mpack_output = assert(io.open(mpack_outputf, 'wb'))
local mpack_output = assert(io.open(eval_funcs_metadata_outputf, 'wb'))
mpack_output:write(vim.mpack.encode(functions))
mpack_output:close()

83
src/gen/gen_api_metadata.lua Normal file
View File

@@ -0,0 +1,83 @@
local mpack = vim.mpack
assert(#arg == 7)
local funcs_metadata_inputf = arg[1] -- exported functions metadata
local ui_metadata_inputf = arg[2] -- ui events metadata
local ui_options_inputf = arg[3] -- for ui options
local git_version_inputf = arg[4] -- git version header
local nvim_version_inputf = arg[5] -- nvim version
local dump_bin_array_inputf = arg[6]
local api_metadata_outputf = arg[7]
local version = loadfile(nvim_version_inputf)()
local git_version = io.open(git_version_inputf):read '*a'
local version_build = git_version:match('#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL
local text = io.open(ui_options_inputf):read '*a'
local ui_options_text = text:match('ui_ext_names%[][^{]+{([^}]+)}')
local ui_options = { 'rgb' }
for x in ui_options_text:gmatch('"([a-z][a-z_]+)"') do
table.insert(ui_options, x)
end
local pieces = {} --- @type string[]
-- Naively using mpack.encode({foo=x, bar=y}) will make the build
-- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead
local function fixdict(num)
if num > 15 then
error 'implement more dict codes'
end
pieces[#pieces + 1] = string.char(128 + num)
end
local function put(item, item2)
table.insert(pieces, mpack.encode(item))
if item2 ~= nil then
table.insert(pieces, mpack.encode(item2))
end
end
fixdict(6)
put('version')
fixdict(1 + #version)
for _, item in ipairs(version) do
-- NB: all items are mandatory. But any error will be less confusing
-- with placeholder vim.NIL (than invalid mpack data)
local val = item[2] == nil and vim.NIL or item[2]
put(item[1], val)
end
put('build', version_build)
put('functions')
table.insert(pieces, io.open(funcs_metadata_inputf, 'rb'):read('*all'))
put('ui_events')
table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all'))
put('ui_options', ui_options)
put('error_types')
fixdict(2)
put('Exception', { id = 0 })
put('Validation', { id = 1 })
put('types')
local types =
{ { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } }
fixdict(#types)
for i, item in ipairs(types) do
put(item[1])
fixdict(2)
put('id', i - 1)
put('prefix', item[2])
end
local packed = table.concat(pieces)
--- @type fun(api_metadata: file*, name: string, packed: string)
local dump_bin_array = loadfile(dump_bin_array_inputf)()
-- serialize the API metadata using msgpack and embed into the resulting
-- binary for easy querying by clients
local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb'))
dump_bin_array(api_metadata_output, 'packed_api_metadata', packed)
api_metadata_output:close()

26
src/gen/gen_steps.zig
View File

@@ -100,32 +100,40 @@ pub fn nvim_gen_sources(
break :ui_step ui_metadata;
};
const funcs_metadata = api_step: {
const eval_funcs_metadata, const exported_funcs_metadata = dispatch_step: {
const gen_step = b.addRunArtifact(nlua0);
gen_step.addFileArg(b.path("src/gen/gen_api_dispatch.lua"));
_ = try gen_header_with_header(b, gen_step, "api/private/dispatch_wrappers.generated.h", nlua0, gen_headers);
_ = gen_header(b, gen_step, "api/private/api_metadata.generated.h", gen_headers);
const funcs_metadata = gen_step.addOutputFileArg("funcs_metadata.mpack");
const exported_funcs_metadata = gen_step.addOutputFileArg("exported_funcs_metadata.mpack");
const eval_funcs_metadata = gen_step.addOutputFileArg("eval_funcs_metadata.mpack");
_ = gen_header(b, gen_step, "lua_api_c_bindings.generated.h", gen_headers);
_ = gen_header(b, gen_step, "keysets_defs.generated.h", gen_headers);
gen_step.addFileArg(ui_metadata);
gen_step.addFileArg(versiondef_git);
gen_step.addFileArg(version_lua);
gen_step.addFileArg(b.path("src/gen/dump_bin_array.lua"));
gen_step.addFileArg(b.path("src/nvim/api/dispatch_deprecated.lua"));
// now follows all .h files with exported functions
for (api_headers.items) |h| {
gen_step.addFileArg(h);
}
break :api_step funcs_metadata;
break :dispatch_step .{ eval_funcs_metadata, exported_funcs_metadata };
};
{
const gen_step = b.addRunArtifact(nlua0);
gen_step.addFileArg(b.path("src/gen/gen_api_metadata.lua"));
gen_step.addFileArg(exported_funcs_metadata);
gen_step.addFileArg(ui_metadata);
gen_step.addFileArg(b.path("src/nvim/api/ui.h"));
gen_step.addFileArg(versiondef_git);
gen_step.addFileArg(version_lua);
gen_step.addFileArg(b.path("src/gen/dump_bin_array.lua"));
_ = gen_header(b, gen_step, "api/private/api_metadata.generated.h", gen_headers);
}
const funcs_data = eval_step: {
const gen_step = b.addRunArtifact(nlua0);
gen_step.addFileArg(b.path("src/gen/gen_eval.lua"));
_ = gen_header(b, gen_step, "funcs.generated.h", gen_headers);
gen_step.addFileArg(funcs_metadata);
gen_step.addFileArg(eval_funcs_metadata);
const funcs_data = gen_step.addOutputFileArg("funcs_data.mpack");
gen_step.addFileArg(b.path("src/nvim/eval.lua"));
break :eval_step funcs_data;

40
src/nvim/CMakeLists.txt
View File

@@ -285,7 +285,8 @@ endif()
# Variables
#-------------------------------------------------------------------------------
set(FUNCS_METADATA ${PROJECT_BINARY_DIR}/funcs_metadata.mpack)
set(EVAL_FUNCS_METADATA ${PROJECT_BINARY_DIR}/eval_funcs_metadata.mpack)
set(EXPORTED_FUNCS_METADATA ${PROJECT_BINARY_DIR}/exported_funcs_metadata.mpack)
set(UI_METADATA ${PROJECT_BINARY_DIR}/ui_metadata.mpack)
set(BINARY_LIB_DIR ${PROJECT_BINARY_DIR}/lib/nvim)
set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
@@ -298,6 +299,7 @@ set(NVIM_RUNTIME_DIR ${PROJECT_SOURCE_DIR}/runtime)
# GENERATOR_DIR
set(API_DISPATCH_GENERATOR ${GENERATOR_DIR}/gen_api_dispatch.lua)
set(API_UI_EVENTS_GENERATOR ${GENERATOR_DIR}/gen_api_ui_events.lua)
set(API_METADATA_GENERATOR ${GENERATOR_DIR}/gen_api_metadata.lua)
set(CHAR_BLOB_GENERATOR ${GENERATOR_DIR}/gen_char_blob.lua)
set(EVENTS_GENERATOR ${GENERATOR_DIR}/gen_events.lua)
set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua)
@@ -564,16 +566,13 @@ set(NVIM_VERSION_LUA ${PROJECT_BINARY_DIR}/nvim_version.lua)
configure_file(${GENERATOR_DIR}/nvim_version.lua.in ${NVIM_VERSION_LUA})
add_custom_command(
OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_API_METADATA}
${FUNCS_METADATA} ${LUA_API_C_BINDINGS} ${GENERATED_KEYSETS_DEFS}
OUTPUT ${GENERATED_API_DISPATCH} ${EXPORTED_FUNCS_METADATA}
${EVAL_FUNCS_METADATA} ${LUA_API_C_BINDINGS} ${GENERATED_KEYSETS_DEFS}
COMMAND ${LUA_GEN} ${API_DISPATCH_GENERATOR}
${GENERATED_API_DISPATCH}
${GENERATED_API_METADATA} ${FUNCS_METADATA}
${EXPORTED_FUNCS_METADATA} ${EVAL_FUNCS_METADATA}
${LUA_API_C_BINDINGS}
${GENERATED_KEYSETS_DEFS}
${UI_METADATA}
${NVIM_VERSION_GIT_H} ${NVIM_VERSION_LUA}
${GENERATOR_DIR}/dump_bin_array.lua
${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
${API_HEADERS}
@@ -584,11 +583,27 @@ add_custom_command(
${API_DISPATCH_GENERATOR}
${GENERATOR_C_GRAMMAR}
${GENERATOR_HASHY}
${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
)
add_custom_command(
OUTPUT ${GENERATED_API_METADATA}
COMMAND ${LUA_GEN} ${API_METADATA_GENERATOR}
${EXPORTED_FUNCS_METADATA}
${UI_METADATA}
${CMAKE_CURRENT_LIST_DIR}/api/ui.h
${NVIM_VERSION_GIT_H} ${NVIM_VERSION_LUA}
${GENERATOR_DIR}/dump_bin_array.lua
${GENERATED_API_METADATA}
DEPENDS
${LUA_GEN_DEPS}
${API_METADATA_GENERATOR}
${GENERATOR_DIR}/dump_bin_array.lua
${UI_METADATA}
${EXPORTED_FUNCS_METADATA}
${NVIM_VERSION_LUA}
${NVIM_VERSION_GIT_H}
${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
)
add_custom_command(
@@ -654,8 +669,8 @@ add_custom_command(OUTPUT ${GENERATED_EX_CMDS_ENUM} ${GENERATED_EX_CMDS_DEFS}
)
add_custom_command(OUTPUT ${GENERATED_FUNCS} ${FUNCS_DATA}
COMMAND ${LUA_GEN} ${FUNCS_GENERATOR} ${GENERATED_FUNCS} ${FUNCS_METADATA} ${FUNCS_DATA} ${CMAKE_CURRENT_LIST_DIR}/eval.lua
DEPENDS ${LUA_GEN_DEPS} ${FUNCS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/eval.lua ${FUNCS_METADATA}
COMMAND ${LUA_GEN} ${FUNCS_GENERATOR} ${GENERATED_FUNCS} ${EVAL_FUNCS_METADATA} ${FUNCS_DATA} ${CMAKE_CURRENT_LIST_DIR}/eval.lua
DEPENDS ${LUA_GEN_DEPS} ${FUNCS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/eval.lua ${EVAL_FUNCS_METADATA}
)
add_custom_command(OUTPUT ${GENERATED_EVENTS_ENUM} ${GENERATED_EVENTS_NAMES_MAP}
@@ -690,6 +705,7 @@ list(APPEND NVIM_GENERATED_FOR_SOURCES
"${GENERATED_OPTIONS}"
"${GENERATED_OPTIONS_MAP}"
"${VIM_MODULE_FILE}"
"${GENERATED_API_METADATA}"
"${PROJECT_BINARY_DIR}/cmake.config/auto/pathdef.h"
)
@@ -960,10 +976,10 @@ add_target(doc-vim
)
add_target(doc-eval
COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${FUNCS_METADATA} ${PROJECT_BINARY_DIR}/runtime/doc/tags
COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${EVAL_FUNCS_METADATA} ${PROJECT_BINARY_DIR}/runtime/doc/tags
DEPENDS
nvim
${FUNCS_METADATA}
${EVAL_FUNCS_METADATA}
${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua
${PROJECT_SOURCE_DIR}/src/nvim/eval.lua
${PROJECT_SOURCE_DIR}/src/nvim/options.lua