import os import json import re import glob import copy import argparse from packaging.version import Version from functools import reduce from pathlib import Path ### # DuckDB C API header generation ### # This script generates the DuckDB C API headers. # # The script works by parsing the definition files in `src/include/duckdb/main/capi/header_generation`, which are then # used to generate the 3 header files: # # The main C header. This is what to include when linking with DuckDB through the C API. DUCKDB_HEADER_OUT_FILE_NAME = 'duckdb.h' DUCKDB_HEADER_OUT_FILE = 'src/include/' + DUCKDB_HEADER_OUT_FILE_NAME # The header to be included by DuckDB C extensions. DUCKDB_HEADER_C_OUT_FILE = 'src/include/duckdb_extension.h' # The header to be included by DuckDB Go extensions. DUCKDB_HEADER_GO_OUT_FILE = 'src/include/duckdb_go_extension.h' # An internal header for DuckDB C extensions. DUCKDB_HEADER_EXT_INTERNAL_OUT_FILE = 'src/include/duckdb/main/capi/extension_api.hpp' # Whether the script allows functions with parameters without a comment explaining them ALLOW_UNCOMMENTED_PARAMS = True DUCKDB_EXT_API_VAR_NAME = 'duckdb_ext_api' DUCKDB_EXT_API_STRUCT_TYPENAME = 'duckdb_ext_api_v1' DEV_VERSION_TAG = 'unstable' # Define the extension struct EXT_API_DEFINITION_PATTERN = 'src/include/duckdb/main/capi/header_generation/apis/v1/*/*.json' EXT_API_EXCLUSION_FILE = 'src/include/duckdb/main/capi/header_generation/apis/v1/exclusion_list.json' # The JSON files that define all available CAPI functions CAPI_FUNCTION_DEFINITION_FILES = 'src/include/duckdb/main/capi/header_generation/functions/**/*.json' # The original order of the function groups in the duckdb.h files. We maintain this for easier PR reviews. # TODO: replace this with alphabetical ordering in a separate PR ORIGINAL_FUNCTION_GROUP_ORDER = [ 'open_connect', 'configuration', 'error_data', 'query_execution', 'safe_fetch_functions', 'helpers', 'date_time_timestamp_helpers', 'hugeint_and_uhugeint_helpers', 'decimal_helpers', 'prepared_statements', 'bind_values_to_prepared_statements', 'execute_prepared_statements', 'extract_statements', 'pending_result_interface', 'value_interface', 'logical_type_interface', 'data_chunk_interface', 'vector_interface', 'validity_mask_functions', 'scalar_functions', 'selection_vector_interface', 'aggregate_functions', 'table_functions', 'table_function_bind', 'table_function_init', 'table_function', 'replacement_scans', 'profiling_info', 'appender', 'table_description', 'arrow_interface', 'threading_information', 'streaming_result_interface', 'cast_functions', 'expression_interface', 'file_system_interface', 'config_options_interface', 'copy_functions', 'catalog_interface', 'logging', ] # The file that forms the base for the header generation BASE_HEADER_TEMPLATE = 'src/include/duckdb/main/capi/header_generation/header_base.hpp.template' # The comment marking where this script will inject its contents BASE_HEADER_CONTENT_MARK = '// FILE_CONTENT_SECTION\n' def HEADER(file): return f'''//===----------------------------------------------------------------------===// // // DuckDB // // {file} // // //===----------------------------------------------------------------------===// // // !!!!!!! // WARNING: this file is autogenerated by scripts/generate_c_api.py, manual changes will be overwritten // !!!!!!! ''' def COMMENT_HEADER(name): return f'''//===--------------------------------------------------------------------===// // {name} //===--------------------------------------------------------------------===// ''' HELPER_MACROS = f''' #ifdef __cplusplus #define DUCKDB_EXTENSION_EXTERN_C_GUARD_OPEN extern "C" {{ #define DUCKDB_EXTENSION_EXTERN_C_GUARD_CLOSE }} #else #define DUCKDB_EXTENSION_EXTERN_C_GUARD_OPEN #define DUCKDB_EXTENSION_EXTERN_C_GUARD_CLOSE #endif #define DUCKDB_EXTENSION_GLUE_HELPER(x, y) x##y #define DUCKDB_EXTENSION_GLUE(x, y) DUCKDB_EXTENSION_GLUE_HELPER(x, y) #define DUCKDB_EXTENSION_STR_HELPER(x) #x #define DUCKDB_EXTENSION_STR(x) DUCKDB_EXTENSION_STR_HELPER(x) #define DUCKDB_EXTENSION_SEMVER_STRING(major, minor, patch) "v" DUCKDB_EXTENSION_STR_HELPER(major) "." DUCKDB_EXTENSION_STR_HELPER(minor) "." DUCKDB_EXTENSION_STR_HELPER(patch) ''' DUCKDB_H_HEADER = HEADER(DUCKDB_HEADER_OUT_FILE_NAME) DUCKDB_C_H_HEADER = HEADER('duckdb_extension.h') DUCKDB_GO_H_HEADER = HEADER('duckdb_go_extension.h') DUCKDB_EXT_INTERNAL_H_HEADER = HEADER('extension_api.hpp') UNSTABLE_WARNING = "\n// WARNING: this API is not yet stable, meaning it is only guaranteed to work for this specific DuckDB version.\n\n" DUCKDB_C_H_HEADER += UNSTABLE_WARNING DUCKDB_GO_H_HEADER += UNSTABLE_WARNING # Loads the header file template. def fetch_header_template_main(): # Read the base header with open(BASE_HEADER_TEMPLATE, 'r') as f: result = f.read() # Trim the base header header_mark = '// DUCKDB_START_OF_HEADER\n' if header_mark not in result: print(f"Could not find the header start mark: {header_mark}") exit(1) return result[result.find(header_mark) + len(header_mark) :] # Includes the header file template. def fetch_header_template_ext(header): return f"""#pragma once #include "{header}" """ # Parse the CAPI_FUNCTION_DEFINITION_FILES to get the full list of functions def parse_capi_function_definitions(): # Collect all functions function_files = glob.glob(CAPI_FUNCTION_DEFINITION_FILES, recursive=True) function_groups = [] function_map = {} # Read functions for file in function_files: with open(file, 'r') as f: try: json_data = json.loads(f.read()) except json.decoder.JSONDecodeError as err: print(f"Invalid JSON found in {file}: {err}") exit(1) function_groups.append(json_data) for function in json_data['entries']: if function['name'] in function_map: print(f"Duplicate symbol found when parsing C API file {file}: {function['name']}") exit(1) function['group'] = json_data['group'] if 'deprecated' in json_data: function['group_deprecated'] = json_data['deprecated'] function_map[function['name']] = function # Reorder to match original order: purely intended to keep the PR review sane function_groups_ordered = [] if len(function_groups) != len(ORIGINAL_FUNCTION_GROUP_ORDER): print( "The list used to match the original order of function groups in the original the duckdb.h file does not match the new one. Did you add a new function group? please also add it to ORIGINAL_FUNCTION_GROUP_ORDER for now." ) for order_group in ORIGINAL_FUNCTION_GROUP_ORDER: curr_group = next(group for group in function_groups if group['group'] == order_group) function_groups.remove(curr_group) function_groups_ordered.append(curr_group) return function_groups_ordered, function_map # Read extension API definitions. def parse_ext_api_definitions(ext_api_definition): api_definitions = {} versions = [] dev_versions = [] for file in list(glob.glob(ext_api_definition)): with open(file, 'r') as f: try: obj = json.loads(f.read()) api_definitions[obj['version']] = obj if obj['version'].startswith("unstable_"): dev_versions.append(obj['version']) else: if Path(file).stem != obj['version']: print( f"\nMismatch between filename and version in file for {file}. Note that unstable versions should have a version starting with 'unstable_' and that stable versions should have the version as their filename" ) exit(1) versions.append(obj['version']) except json.decoder.JSONDecodeError as err: print(f"\nInvalid JSON found in {file}: {err}") exit(1) versions.sort(key=Version) dev_versions.sort() return [api_definitions[x] for x in (versions + dev_versions)] # Decode all excluded functions and return them. def parse_exclusion_list(function_map): exclusion_set = set() with open(EXT_API_EXCLUSION_FILE, 'r') as f: try: data = json.loads(f.read()) except json.decoder.JSONDecodeError as err: print(f"\nInvalid JSON found in {EXT_API_EXCLUSION_FILE}: {err}") exit(1) for group in data['exclusion_list']: for entry in group['entries']: if entry not in function_map: print(f"\nInvalid item found in exclusion list: {entry}. This entry does not occur in the API!") exit(1) exclusion_set.add(entry) return exclusion_set # Creates the comment that accompanies a C API function. def create_function_comment(function_obj): result = '' function_name = function_obj['name'] # Construct comment if 'comment' in function_obj: comment = function_obj['comment'] result += '/*!\n' result += comment['description'] # result += '\n\n' if 'params' in function_obj: for param in function_obj['params']: param_name = param['name'] if not 'param_comments' in comment: if not ALLOW_UNCOMMENTED_PARAMS: print(comment) print(f'\nMissing param comments for function {function_name}') exit(1) continue if param['name'] in comment['param_comments']: param_comment = comment['param_comments'][param['name']] result += f'* @param {param_name} {param_comment}\n' elif not ALLOW_UNCOMMENTED_PARAMS: print(f'\nUncommented parameter found: {param_name} of function {function_name}') exit(1) if 'return_value' in comment: comment_return_value = comment['return_value'] result += f'* @return {comment_return_value}\n' result += '*/\n' return result # Creates the function declaration for the regular C header file def create_function_declaration(function_obj): result = '' function_name = function_obj['name'] function_return_type = function_obj['return_type'] # Construct function declaration result += f'DUCKDB_C_API {function_return_type}' if result[-1] != '*': result += ' ' result += f'{function_name}(' if 'params' in function_obj: if len(function_obj['params']) > 0: for param in function_obj['params']: param_type = param['type'] param_name = param['name'] result += f'{param_type}' if result[-1] != '*': result += ' ' result += f'{param_name}, ' result = result[:-2] # Trailing comma result += ');\n' return result # Creates the function declaration for extension api struct def create_struct_member(function_obj): result = '' function_name = function_obj['name'] function_return_type = function_obj['return_type'] result += f' {function_return_type} (*{function_name})(' if 'params' in function_obj: if len(function_obj['params']) > 0: for param in function_obj['params']: param_type = param['type'] param_name = param['name'] result += f'{param_type} {param_name},' result = result[:-1] # Trailing comma result += ');' return result # Creates the function declaration for extension api struct def create_function_typedef(function_obj): function_name = function_obj['name'] return f'#define {function_name} {DUCKDB_EXT_API_VAR_NAME}.{function_name}\n' def headline_capitalize(s, i): if i > 0 and s in [ "a", "an", "the", "and", "but", "or", "nor", "for", "so", "yet", "about", "above", "across", "after", "against", "along", "among", "around", "at", "before", "behind", "below", "beneath", "beside", "between", "beyond", "by", "despite", "down", "during", "except", "for", "from", "in", "inside", "into", "like", "near", "of", "off", "on", "onto", "out", "outside", "over", "past", "since", "through", "throughout", "to", "toward", "under", "underneath", "until", "up", "upon", "with", "within", "without", ]: return s else: return s.capitalize() def to_camel_case(snake_str): return " ".join(headline_capitalize(s, i) for i, s in enumerate(snake_str.lower().split("_"))) def parse_semver(version): if version[0] != 'v': raise Exception(f"\nVersion string {version} does not start with a v") versions = version[1:].split(".") if len(versions) != 3: print(f"\nVersion string {version} is invalid, only vx.y.z is supported") exit(1) return int(versions[0]), int(versions[1]), int(versions[2]) def create_version_defines(version): major, minor, patch = parse_semver(version) version_string = f"v{major}.{minor}.{patch}" result = "" result += f"#define DUCKDB_EXTENSION_API_VERSION_MAJOR {major}\n" result += f"#define DUCKDB_EXTENSION_API_VERSION_MINOR {minor}\n" result += f"#define DUCKDB_EXTENSION_API_VERSION_PATCH {patch}\n" result += f"#define DUCKDB_EXTENSION_API_VERSION_STRING \"{version_string}\"\n" return result def comment_function_group(name, deprecated, use_instead, description): text = '//----------------------------------------------------------------------------------------------------------------------' text += '\n// ' + name text += '\n//----------------------------------------------------------------------------------------------------------------------' text += '\n// DESCRIPTION:' text += '\n// ' + description if deprecated: text += '\n//\n// DEPRECATION NOTICE:\n// This function group is deprecated and scheduled for removal.' if use_instead != "": text += '\n//' text += '\n// USE INSTEAD:' text += '\n//' + use_instead else: print(f"Function group {name} marked as deprecated without 'use_instead' instructions!") exit(1) elif use_instead != "": print(f"Function group {name} not marked as deprecated but has 'use_instead' instructions!") exit(1) text += '\n//----------------------------------------------------------------------------------------------------------------------\n\n' return text # Create duckdb.h def create_duckdb_h(file, function_groups, write_functions=True): declarations = '' if write_functions: declarations += f'#ifndef DUCKDB_API_EXCLUDE_FUNCTIONS\n\n' declarations += COMMENT_HEADER("Functions") declarations += '\n' for curr_group in function_groups: group_name = to_camel_case(curr_group['group']) group_is_deprecated = 'deprecated' in curr_group and curr_group['deprecated'] use_instead = "" if 'use_instead' in curr_group: use_instead = curr_group['use_instead'] if not 'description' in curr_group: raise Exception(f"Group {group_name} does not have a description!") description = curr_group['description'] declarations += comment_function_group(group_name, group_is_deprecated, use_instead, description) declarations += '\n' deprecated_state = False if group_is_deprecated: declarations += f'#ifndef DUCKDB_API_NO_DEPRECATED\n' deprecated_state = True for function in curr_group['entries']: function_is_deprecated = group_is_deprecated or ('deprecated' in function and function['deprecated']) if deprecated_state and not function_is_deprecated: declarations += '#endif\n\n' deprecated_state = False elif not deprecated_state and function_is_deprecated: declarations += '#ifndef DUCKDB_API_NO_DEPRECATED\n' deprecated_state = True function_comment = create_function_comment(function) function_deprecation_notice = ( '**DEPRECATED**' in function_comment or '**DEPRECATION NOTICE**' in function_comment ) if function_is_deprecated and not function_deprecation_notice: raise Exception( f"Function {str(function)} is labeled as deprecated but the comment does not indicate this" ) elif not function_is_deprecated and function_deprecation_notice: raise Exception( f"Function {str(function)} is not labeled as deprecated but the comment indicates that it is" ) declarations += function_comment declarations += create_function_declaration(function) declarations += '\n' if deprecated_state: declarations += '#endif\n\n' declarations += '#endif\n' # Write the function declarations to the header template, and then write the file. header_template = fetch_header_template_main() duckdb_h = DUCKDB_H_HEADER + header_template.replace(BASE_HEADER_CONTENT_MARK, declarations) with open(file, 'w+') as f: f.write(duckdb_h) def write_struct_member_definitions(function_map, version_entries, initialize=False): result = "" if initialize: for function_name in version_entries: function_lookup = function_map[function_name] function_lookup_name = function_lookup['name'] result += f' result.{function_lookup_name} = {function_lookup_name};\n' elif len(version_entries) > 0: count = len(version_entries) first_function = version_entries[0] result += f' memset(&result.{first_function}, 0, sizeof(result.{first_function}) * {count});\n' return result # Creates function pointer invokers. def create_struct_member_invoker(function_obj): result = '' function_name = function_obj['name'] function_return_type = function_obj['return_type'] # Construct typedef declaration result += f'static {function_return_type}' if result[-1] != '*': result += ' ' result += f'{function_name}(' invoked_params = '' if 'params' in function_obj: if len(function_obj['params']) > 0: for param in function_obj['params']: param_type = param['type'] param_name = param['name'] result += f'{param_type}' if result[-1] != '*': result += ' ' result += f'{param_name}, ' invoked_params += f'{param_name}, ' result = result[:-2] # Trailing comma result += ') {\n' if invoked_params != '': invoked_params = invoked_params[:-2] # Trailing comma result += f'\treturn duckdb_ext_api->{function_name}({invoked_params}); \n' result += '}' return result def create_struct_version_defines(api_definition): result = "//! These defines control which version of the API is available\n" for i in range(1, len(api_definition) + 1): current_version = api_definition[-i]['version'] if i < len(api_definition): prev_version = api_definition[-(i + 1)]['version'] else: prev_version = None print(f"current: {current_version} prev: {prev_version}") if prev_version: prev_major, prev_minor, prev_patch = parse_semver(prev_version) if current_version == DEV_VERSION_TAG: result += "#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE\n" result += f"#define DUCKDB_EXTENSION_API_VERSION_{prev_major}_{prev_minor}_{prev_patch}\n" result += "#endif\n" else: major, minor, patch = parse_semver(current_version) result += f"#ifdef DUCKDB_EXTENSION_API_VERSION_{major}_{minor}_{patch}\n" result += f"#define DUCKDB_EXTENSION_API_VERSION_{prev_major}_{prev_minor}_{prev_patch}\n" result += "#endif\n" result += "\n" first_version = api_definition[0]['version'] first_major, first_minor, first_patch = parse_semver(first_version) result += f"// No version was explicitly set, we assume latest\n" result += f"#ifndef DUCKDB_EXTENSION_API_VERSION_{first_major}_{first_minor}_{first_patch}\n" result += f"#define DUCKDB_EXTENSION_API_VERSION_LATEST\n" result += f"#endif\n" result += "\n" return result # Generate the struct without any other content. def generate_basic_extension_struct(function_map, api_definition, add_version_defines): functions_in_struct = set() extension_struct_finished = COMMENT_HEADER("Function pointer struct") extension_struct_finished += 'typedef struct {\n' for api_version_entry in api_definition: if len(api_version_entry['entries']) == 0: continue version = api_version_entry['version'] if version.startswith("unstable_"): if 'description' in api_version_entry: extension_struct_finished += f"// {api_version_entry['description']}\n" if add_version_defines: extension_struct_finished += f"#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE\n" else: extension_struct_finished += f"\n" else: if add_version_defines: major, minor, patch = parse_semver(version) extension_struct_finished += f"#if DUCKDB_EXTENSION_API_VERSION_MINOR > {minor} || (DUCKDB_EXTENSION_API_VERSION_MINOR == {minor} && DUCKDB_EXTENSION_API_VERSION_PATCH >= {patch}) // {version}\n" else: extension_struct_finished += f"// {version}\n" for function_name in api_version_entry['entries']: function_lookup = function_map[function_name] functions_in_struct.add(function_lookup['name']) extension_struct_finished += create_struct_member(function_lookup) extension_struct_finished += '\n' if add_version_defines: extension_struct_finished += "#endif\n\n" extension_struct_finished += '} ' + f'{DUCKDB_EXT_API_STRUCT_TYPENAME};\n\n' return extension_struct_finished, functions_in_struct # Check for missing and duplicate entries. def struct_validate_exclusion_list(functions_in_struct, api_definition, exclusion_set, function_groups): missing_entries = [] for group in function_groups: for function in group['entries']: if function['name'] not in functions_in_struct and function['name'] not in exclusion_set: missing_entries.append(function['name']) if missing_entries: print( "\nExclusion list validation failed! This means a C API function has been defined but not added to the API struct nor the exclusion list" ) print(f" * Missing functions: {missing_entries}") exit(1) double_entries = [] for api_version_entry in api_definition: for function_name in api_version_entry['entries']: if function_name in exclusion_set: double_entries.append(function_name) if double_entries: print( "\nExclusion list is invalid, there are entries in the extension api that are also in the exclusion list!" ) print(f" * Missing functions: {double_entries}") exit(1) def create_extension_api_struct( function_groups, function_map, api_definition, exclusion_set, with_create_method=False, with_member_invoker=False, add_version_defines=False, create_method_name='', validate_exclusion_list=True, ): extension_struct_finished, functions_in_struct = generate_basic_extension_struct( function_map, api_definition, add_version_defines ) if validate_exclusion_list: struct_validate_exclusion_list(functions_in_struct, api_definition, exclusion_set, function_groups) if with_create_method: extension_struct_finished += COMMENT_HEADER("Struct Create Method") extension_struct_finished += f"inline {DUCKDB_EXT_API_STRUCT_TYPENAME} {create_method_name}() {{\n" extension_struct_finished += f" {DUCKDB_EXT_API_STRUCT_TYPENAME} result;\n" for api_version_entry in api_definition: if len(api_version_entry['entries']) == 0: continue extension_struct_finished += write_struct_member_definitions( function_map, api_version_entry['entries'], initialize=True ) extension_struct_finished += " return result;\n" extension_struct_finished += "}\n\n" if with_member_invoker: extension_struct_finished += f'extern {DUCKDB_EXT_API_STRUCT_TYPENAME} *{DUCKDB_EXT_API_VAR_NAME};\n\n' extension_struct_finished += COMMENT_HEADER("Invoker Functions") for api_version_entry in api_definition: if len(api_version_entry['entries']) == 0: continue extension_struct_finished += '\n' version = api_version_entry['version'] if version.startswith("unstable_"): if add_version_defines: extension_struct_finished += f"#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE\n" else: extension_struct_finished += f"\n" else: if add_version_defines: major, minor, patch = parse_semver(version) extension_struct_finished += f"#if DUCKDB_EXTENSION_API_VERSION_MINOR > {minor} || (DUCKDB_EXTENSION_API_VERSION_MINOR == {minor} && DUCKDB_EXTENSION_API_VERSION_PATCH >= {patch}) // {version}\n" else: extension_struct_finished += f"// {version}\n" extension_struct_finished += '\n' for function_name in api_version_entry['entries']: function_lookup = function_map[function_name] functions_in_struct.add(function_lookup['name']) extension_struct_finished += create_struct_member_invoker(function_lookup) extension_struct_finished += '\n\n' if add_version_defines: extension_struct_finished += "#endif\n\n" return extension_struct_finished def create_duckdb_ext_h_versioning( ext_api_version, function_groups, api_struct_definition, exclusion_set, duckdb_header, duckdb_ext_header, with_member_invoker=False, ): # Create the versioning defines major, minor, patch = parse_semver(ext_api_version) versioning_defines = f"""//! Set version to latest if no explicit version is defined #if !defined(DUCKDB_EXTENSION_API_VERSION_MAJOR) && !defined(DUCKDB_EXTENSION_API_VERSION_MINOR) && !defined(DUCKDB_EXTENSION_API_VERSION_PATCH) #define DUCKDB_EXTENSION_API_VERSION_MAJOR {major} #define DUCKDB_EXTENSION_API_VERSION_MINOR {minor} #define DUCKDB_EXTENSION_API_VERSION_PATCH {patch} #elif !(defined(DUCKDB_EXTENSION_API_VERSION_MAJOR) && defined(DUCKDB_EXTENSION_API_VERSION_MINOR) && defined(DUCKDB_EXTENSION_API_VERSION_PATCH)) #error "either all or none of the DUCKDB_EXTENSION_API_VERSION_ defines should be defined" #endif //! Set the DUCKDB_EXTENSION_API_VERSION_STRING which is passed to DuckDB on extension load #ifdef DUCKDB_EXTENSION_API_UNSTABLE_VERSION #define DUCKDB_EXTENSION_API_VERSION_STRING DUCKDB_EXTENSION_API_UNSTABLE_VERSION #else #define DUCKDB_EXTENSION_API_VERSION_STRING DUCKDB_EXTENSION_SEMVER_STRING(DUCKDB_EXTENSION_API_VERSION_MAJOR, DUCKDB_EXTENSION_API_VERSION_MINOR, DUCKDB_EXTENSION_API_VERSION_PATCH) #endif #if DUCKDB_EXTENSION_API_VERSION_MAJOR != {major} #error "This version of the extension API header only supports API VERSION v{major}.x.x" #endif """ # Begin constructing the header file duckdb_ext_h = "" duckdb_ext_h += duckdb_ext_header duckdb_ext_h += fetch_header_template_ext(duckdb_header) duckdb_ext_h += COMMENT_HEADER("Util Macros") duckdb_ext_h += HELPER_MACROS duckdb_ext_h += COMMENT_HEADER("Versioning") duckdb_ext_h += versioning_defines duckdb_ext_h += "\n" duckdb_ext_h += create_extension_api_struct( function_groups, function_map, api_struct_definition, exclusion_set, add_version_defines=True, with_member_invoker=with_member_invoker, ) duckdb_ext_h += "\n\n" return duckdb_ext_h # Create duckdb_extension.h def create_duckdb_c_ext_h(file, ext_api_version, function_groups, api_struct_definition, exclusion_set): duckdb_ext_h = create_duckdb_ext_h_versioning( ext_api_version, function_groups, api_struct_definition, exclusion_set, DUCKDB_HEADER_OUT_FILE_NAME, DUCKDB_C_H_HEADER, ) # Generate the typedefs typedefs = "" for api_version_entry in api_struct_definition: version = api_version_entry['version'] # Collect the typedefs for this version grouped_typedefs = [] for group in function_groups: functions_to_add = [] for function in group['entries']: if function['name'] not in api_version_entry['entries']: continue functions_to_add.append(function) if functions_to_add: grouped_typedefs.append((group['group'], functions_to_add)) if len(grouped_typedefs) == 0: continue typedefs += f"// Version {version}\n" for group in grouped_typedefs: for fun_to_add in group[1]: typedefs += create_function_typedef(fun_to_add) typedefs += '\n' duckdb_ext_h += COMMENT_HEADER("Typedefs mapping functions to struct entries") duckdb_ext_h += typedefs duckdb_ext_h += "\n" # Add The struct global macros duckdb_ext_h += COMMENT_HEADER("Struct Global Macros") duckdb_ext_h += f"""// This goes in the c/c++ file containing the entrypoint (handle #define DUCKDB_EXTENSION_GLOBAL {DUCKDB_EXT_API_STRUCT_TYPENAME} {DUCKDB_EXT_API_VAR_NAME} = {{0}}; // Initializes the C Extension API: First thing to call in the extension entrypoint #define DUCKDB_EXTENSION_API_INIT(info, access, minimum_api_version) {DUCKDB_EXT_API_STRUCT_TYPENAME} * res = ({DUCKDB_EXT_API_STRUCT_TYPENAME} *)access->get_api(info, minimum_api_version); if (!res) {{return false;}}; {DUCKDB_EXT_API_VAR_NAME} = *res; """ duckdb_ext_h += f""" // Place in global scope of any C/C++ file that needs to access the extension API #define DUCKDB_EXTENSION_EXTERN extern {DUCKDB_EXT_API_STRUCT_TYPENAME} {DUCKDB_EXT_API_VAR_NAME}; #ifdef _WIN32 #define DUCKDB_CAPI_ENTRY_VISIBILITY __declspec(dllexport) #else #define DUCKDB_CAPI_ENTRY_VISIBILITY __attribute__((visibility("default"))) #endif """ # Add the entrypoint macros duckdb_ext_h += COMMENT_HEADER("Entrypoint Macros") duckdb_ext_h += """ // Note: the DUCKDB_EXTENSION_ENTRYPOINT macro requires DUCKDB_EXTENSION_NAME to be set. #ifdef DUCKDB_EXTENSION_NAME // Main entrypoint: opens (and closes) a connection automatically for the extension to register its functionality through #define DUCKDB_EXTENSION_ENTRYPOINT\ DUCKDB_EXTENSION_GLOBAL static bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal)(duckdb_connection connection, duckdb_extension_info info, struct duckdb_extension_access *access);\ DUCKDB_EXTENSION_EXTERN_C_GUARD_OPEN\ DUCKDB_CAPI_ENTRY_VISIBILITY DUCKDB_EXTENSION_API bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api)(\ duckdb_extension_info info, struct duckdb_extension_access *access) {\ DUCKDB_EXTENSION_API_INIT(info, access, DUCKDB_EXTENSION_API_VERSION_STRING);\ duckdb_database *db = access->get_database(info);\ duckdb_connection conn;\ if (duckdb_connect(*db, &conn) == DuckDBError) {\ access->set_error(info, "Failed to open connection to database");\ return false;\ }\ bool init_result = DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal)(conn, info, access);\ duckdb_disconnect(&conn);\ return init_result;\ }\ DUCKDB_EXTENSION_EXTERN_C_GUARD_CLOSE static bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal) // Custom entrypoint: just forwards the info and access #define DUCKDB_EXTENSION_ENTRYPOINT_CUSTOM\ DUCKDB_EXTENSION_GLOBAL static bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal)(\ duckdb_extension_info info, struct duckdb_extension_access *access);\ DUCKDB_EXTENSION_EXTERN_C_GUARD_OPEN\ DUCKDB_CAPI_ENTRY_VISIBILITY DUCKDB_EXTENSION_API bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api)(\ duckdb_extension_info info, struct duckdb_extension_access *access) {\ DUCKDB_EXTENSION_API_INIT(info, access, DUCKDB_EXTENSION_API_VERSION_STRING);\ return DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal)(info, access);\ }\ DUCKDB_EXTENSION_EXTERN_C_GUARD_CLOSE static bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal) #endif """ with open(file, 'w+') as f: f.write(duckdb_ext_h) # Create duckdb_go_extension.h def create_duckdb_go_ext_h(file, ext_api_version, function_groups, api_struct_definition, exclusion_set): duckdb_ext_h = create_duckdb_ext_h_versioning( ext_api_version, function_groups, api_struct_definition, exclusion_set, DUCKDB_HEADER_OUT_FILE_NAME, DUCKDB_GO_H_HEADER, with_member_invoker=True, ) with open(file, 'w+') as f: f.write(duckdb_ext_h) # Create duckdb_extension_internal.hpp def create_duckdb_ext_internal_h(ext_api_version, function_groups, function_map, ext_api_definitions, exclusion_set): duckdb_ext_h = fetch_header_template_ext(DUCKDB_HEADER_OUT_FILE_NAME) duckdb_ext_h += create_extension_api_struct( function_groups, function_map, ext_api_definitions, exclusion_set, with_create_method=True, create_method_name='CreateAPIv1', ) duckdb_ext_h += create_version_defines(ext_api_version) with open(DUCKDB_HEADER_EXT_INTERNAL_OUT_FILE, 'w+') as f: f.write(duckdb_ext_h) def get_extension_api_version(ext_api_definitions): latest_version = "" for version_entry in ext_api_definitions: if version_entry['version'].startswith('v'): latest_version = version_entry['version'] if version_entry['version'].startswith("unstable_"): break return latest_version def create_struct_function_set(api_definitions): result = set() for api in api_definitions: for entry in api['entries']: if entry in result: raise Exception(f"Duplicate entry found for function '{entry}'!") result.add(entry) return result if __name__ == "__main__": arg_parser = argparse.ArgumentParser(description='Generate the C API') arg_parser.add_argument( '-go', '--generate-go', type=bool, help='Emit the duckdb go extension header', required=False, default=False ) args = arg_parser.parse_args() # parse the api definition (which fields make it into the struct) ext_api_definitions = parse_ext_api_definitions(EXT_API_DEFINITION_PATTERN) # extract a set of the function names and the latest version of the api definition ext_api_set = create_struct_function_set(ext_api_definitions) ext_api_version = get_extension_api_version(ext_api_definitions) function_groups, function_map = parse_capi_function_definitions() function_map_size = len(function_map) api_struct_function_count = sum([len(y['entries']) for y in ext_api_definitions]) ext_api_exclusion_set = parse_exclusion_list(function_map) ext_api_exclusion_set_size = len(ext_api_exclusion_set) print("Information") print(f" * Current Extension C API Version: {ext_api_version}") print(f" * Total functions: {function_map_size}") print(f" * Functions in C API struct: {api_struct_function_count}") print(f" * Functions in C API but excluded from struct: {ext_api_exclusion_set_size}") print() print("Generating headers") print(f" * {DUCKDB_HEADER_OUT_FILE}") create_duckdb_h(DUCKDB_HEADER_OUT_FILE, function_groups) print(f" * {DUCKDB_HEADER_C_OUT_FILE}") create_duckdb_c_ext_h( DUCKDB_HEADER_C_OUT_FILE, ext_api_version, function_groups, ext_api_definitions, ext_api_exclusion_set, ) print(f" * {DUCKDB_HEADER_EXT_INTERNAL_OUT_FILE}") create_duckdb_ext_internal_h( ext_api_version, function_groups, function_map, ext_api_definitions, ext_api_exclusion_set ) print() os.system(f"python3 scripts/format.py {DUCKDB_HEADER_OUT_FILE} --fix --noconfirm") os.system(f"python3 scripts/format.py {DUCKDB_HEADER_C_OUT_FILE} --fix --noconfirm") os.system(f"python3 scripts/format.py {DUCKDB_HEADER_EXT_INTERNAL_OUT_FILE} --fix --noconfirm") print() print("C API headers generated successfully!") print() if args.generate_go: print(f" * {DUCKDB_HEADER_GO_OUT_FILE}") create_duckdb_go_ext_h( DUCKDB_HEADER_GO_OUT_FILE, ext_api_version, function_groups, ext_api_definitions, ext_api_exclusion_set, ) os.system(f"python3 scripts/format.py {DUCKDB_HEADER_GO_OUT_FILE} --fix --noconfirm") print() print("Go C API headers generated successfully!")