From 3b991f81c45720515144b928bb8d1f03577628f1 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson Date: Fri, 17 Nov 2023 11:47:17 +0100 Subject: [PATCH] llvmjit: Use explicit LLVMContextRef for inlining When performing inlining LLVM unfortunately "leaks" types (the types survive and are usable, but a new round of inlining will recreate new structurally equivalent types). This accumulation will over time amount to a memory leak which for some queries can be large enough to trigger the OOM process killer. To avoid accumulation of types, all IR related data is stored in an LLVMContextRef which is dropped and recreated in order to release all types. Dropping and recreating incurs overhead, so it will be done only after 100 queries. This is a heuristic which might be revisited, but until we can get the size of the context from LLVM we are flying a bit blind. This issue has been reported several times, there may be more references to it in the archives on top of the threads linked below. This is a backpatch of 9dce22033d5 to all supported branches. Reported-By: Justin Pryzby Reported-By: Kurt Roeckx Reported-By: Jaime Casanova Reported-By: Lauri Laanmets Author: Andres Freund and Daniel Gustafsson Discussion: https://postgr.es/m/7acc8678-df5f-4923-9cf6-e843131ae89d@www.fastmail.com Discussion: https://postgr.es/m/20201218235607.GC30237@telsasoft.com Discussion: https://postgr.es/m/CAPH-tTxLf44s3CvUUtQpkDr1D8Hxqc2NGDzGXS1ODsfiJ6WSqA@mail.gmail.com Backpatch-through: v12 --- src/backend/jit/llvm/llvmjit.c | 141 +++++++++++++++++++++--- src/backend/jit/llvm/llvmjit_deform.c | 77 ++++++------- src/backend/jit/llvm/llvmjit_expr.c | 72 ++++++------ src/backend/jit/llvm/llvmjit_inline.cpp | 33 ++++-- src/include/jit/llvmjit.h | 8 ++ src/include/jit/llvmjit_emit.h | 30 +++-- 6 files changed, 252 insertions(+), 109 deletions(-) diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c index 8df4e33738..07a720d3e4 100644 --- a/src/backend/jit/llvm/llvmjit.c +++ b/src/backend/jit/llvm/llvmjit.c @@ -51,6 +51,8 @@ #endif +#define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100 + /* Handle of a module emitted via ORC JIT */ typedef struct LLVMJitHandle { @@ -103,8 +105,15 @@ LLVMModuleRef llvm_types_module = NULL; static bool llvm_session_initialized = false; static size_t llvm_generation = 0; + +/* number of LLVMJitContexts that currently are in use */ +static size_t llvm_jit_context_in_use_count = 0; + +/* how many times has the current LLVMContextRef been used */ +static size_t llvm_llvm_context_reuse_count = 0; static const char *llvm_triple = NULL; static const char *llvm_layout = NULL; +static LLVMContextRef llvm_context; static LLVMTargetRef llvm_targetref; @@ -125,6 +134,8 @@ static void llvm_compile_module(LLVMJitContext *context); static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module); static void llvm_create_types(void); +static void llvm_set_target(void); +static void llvm_recreate_llvm_context(void); static uint64_t llvm_resolve_symbol(const char *name, void *ctx); #if LLVM_VERSION_MAJOR > 11 @@ -146,6 +157,63 @@ _PG_jit_provider_init(JitProviderCallbacks *cb) cb->compile_expr = llvm_compile_expr; } + +/* + * Every now and then create a new LLVMContextRef. Unfortunately, during every + * round of inlining, types may "leak" (they can still be found/used via the + * context, but new types will be created the next time in inlining is + * performed). To prevent that from slowly accumulating problematic amounts of + * memory, recreate the LLVMContextRef we use. We don't want to do so too + * often, as that implies some overhead (particularly re-loading the module + * summaries / modules is fairly expensive). A future TODO would be to make + * this more finegrained and only drop/recreate the LLVMContextRef when we know + * there has been inlining. If we can get the size of the context from LLVM + * then that might be a better way to determine when to drop/recreate rather + * then the usagecount heuristic currently employed. + */ +static void +llvm_recreate_llvm_context(void) +{ + if (!llvm_context) + elog(ERROR, "Trying to recreate a non-existing context"); + + /* + * We can only safely recreate the LLVM context if no other code is being + * JITed, otherwise we'd release the types in use for that. + */ + if (llvm_jit_context_in_use_count > 0) + { + llvm_llvm_context_reuse_count++; + return; + } + + if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX) + { + llvm_llvm_context_reuse_count++; + return; + } + + /* + * Need to reset the modules that the inlining code caches before + * disposing of the context. LLVM modules exist within a specific LLVM + * context, therefore disposing of the context before resetting the cache + * would lead to dangling pointers to modules. + */ + llvm_inline_reset_caches(); + + LLVMContextDispose(llvm_context); + llvm_context = LLVMContextCreate(); + llvm_llvm_context_reuse_count = 0; + + /* + * Re-build cached type information, so code generation code can rely on + * that information to be present (also prevents the variables to be + * dangling references). + */ + llvm_create_types(); +} + + /* * Create a context for JITing work. * @@ -162,6 +230,8 @@ llvm_create_context(int jitFlags) llvm_session_initialize(); + llvm_recreate_llvm_context(); + ResourceOwnerEnlargeJIT(CurrentResourceOwner); context = MemoryContextAllocZero(TopMemoryContext, @@ -172,6 +242,8 @@ llvm_create_context(int jitFlags) context->base.resowner = CurrentResourceOwner; ResourceOwnerRememberJIT(CurrentResourceOwner, PointerGetDatum(context)); + llvm_jit_context_in_use_count++; + return context; } @@ -181,7 +253,13 @@ llvm_create_context(int jitFlags) static void llvm_release_context(JitContext *context) { - LLVMJitContext *llvm_context = (LLVMJitContext *) context; + LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context; + + /* + * Consider as cleaned up even if we skip doing so below, that way we can + * verify the tracking is correct (see llvm_shutdown()). + */ + llvm_jit_context_in_use_count--; /* * When this backend is exiting, don't clean up LLVM. As an error might @@ -193,18 +271,18 @@ llvm_release_context(JitContext *context) llvm_enter_fatal_on_oom(); - if (llvm_context->module) + if (llvm_jit_context->module) { - LLVMDisposeModule(llvm_context->module); - llvm_context->module = NULL; + LLVMDisposeModule(llvm_jit_context->module); + llvm_jit_context->module = NULL; } - while (llvm_context->handles != NIL) + while (llvm_jit_context->handles != NIL) { LLVMJitHandle *jit_handle; - jit_handle = (LLVMJitHandle *) linitial(llvm_context->handles); - llvm_context->handles = list_delete_first(llvm_context->handles); + jit_handle = (LLVMJitHandle *) linitial(llvm_jit_context->handles); + llvm_jit_context->handles = list_delete_first(llvm_jit_context->handles); #if LLVM_VERSION_MAJOR > 11 { @@ -232,6 +310,8 @@ llvm_release_context(JitContext *context) pfree(jit_handle); } + list_free(llvm_jit_context->handles); + llvm_jit_context->handles = NIL; llvm_leave_fatal_on_oom(); } @@ -251,7 +331,7 @@ llvm_mutable_module(LLVMJitContext *context) { context->compiled = false; context->module_generation = llvm_generation++; - context->module = LLVMModuleCreateWithName("pg"); + context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context); LLVMSetTarget(context->module, llvm_triple); LLVMSetDataLayout(context->module, llvm_layout); } @@ -835,6 +915,14 @@ llvm_session_initialize(void) LLVMInitializeNativeAsmPrinter(); LLVMInitializeNativeAsmParser(); + if (llvm_context == NULL) + { + llvm_context = LLVMContextCreate(); + + llvm_jit_context_in_use_count = 0; + llvm_llvm_context_reuse_count = 0; + } + /* * When targeting LLVM 15, turn off opaque pointers for the context we * build our code in. We don't need to do so for other contexts (e.g. @@ -854,6 +942,11 @@ llvm_session_initialize(void) */ llvm_create_types(); + /* + * Extract target information from loaded module. + */ + llvm_set_target(); + if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0) { elog(FATAL, "failed to query triple %s", error); @@ -949,6 +1042,10 @@ llvm_shutdown(int code, Datum arg) return; } + if (llvm_jit_context_in_use_count != 0) + elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)", + llvm_jit_context_in_use_count); + #if LLVM_VERSION_MAJOR > 11 { if (llvm_opt3_orc) @@ -1011,6 +1108,23 @@ load_return_type(LLVMModuleRef mod, const char *name) return typ; } +/* + * Load triple & layout from clang emitted file so we're guaranteed to be + * compatible. + */ +static void +llvm_set_target(void) +{ + if (!llvm_types_module) + elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded"); + + if (llvm_triple == NULL) + llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module)); + + if (llvm_layout == NULL) + llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module)); +} + /* * Load required information, types, function signatures from llvmjit_types.c * and make them available in global variables. @@ -1034,19 +1148,12 @@ llvm_create_types(void) } /* eagerly load contents, going to need it all */ - if (LLVMParseBitcode2(buf, &llvm_types_module)) + if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module)) { - elog(ERROR, "LLVMParseBitcode2 of %s failed", path); + elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path); } LLVMDisposeMemoryBuffer(buf); - /* - * Load triple & layout from clang emitted file so we're guaranteed to be - * compatible. - */ - llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module)); - llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module)); - TypeSizeT = llvm_pg_var_type("TypeSizeT"); TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool"); TypeStorageBool = llvm_pg_var_type("TypeStorageBool"); diff --git a/src/backend/jit/llvm/llvmjit_deform.c b/src/backend/jit/llvm/llvmjit_deform.c index e79c2a3fb8..a624a7e461 100644 --- a/src/backend/jit/llvm/llvmjit_deform.c +++ b/src/backend/jit/llvm/llvmjit_deform.c @@ -53,6 +53,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, char *funcname; LLVMModuleRef mod; + LLVMContextRef lc; LLVMBuilderRef b; LLVMTypeRef deform_sig; @@ -115,6 +116,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, return NULL; mod = llvm_mutable_module(context); + lc = LLVMGetModuleContext(mod); funcname = llvm_expand_funcname(context, "deform"); @@ -149,8 +151,8 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, param_types[0] = l_ptr(StructTupleTableSlot); - deform_sig = LLVMFunctionType(LLVMVoidType(), param_types, - lengthof(param_types), 0); + deform_sig = LLVMFunctionType(LLVMVoidTypeInContext(lc), + param_types, lengthof(param_types), 0); } v_deform_fn = LLVMAddFunction(mod, funcname, deform_sig); LLVMSetLinkage(v_deform_fn, LLVMInternalLinkage); @@ -158,17 +160,17 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, llvm_copy_attributes(AttributeTemplate, v_deform_fn); b_entry = - LLVMAppendBasicBlock(v_deform_fn, "entry"); + LLVMAppendBasicBlockInContext(lc, v_deform_fn, "entry"); b_adjust_unavail_cols = - LLVMAppendBasicBlock(v_deform_fn, "adjust_unavail_cols"); + LLVMAppendBasicBlockInContext(lc, v_deform_fn, "adjust_unavail_cols"); b_find_start = - LLVMAppendBasicBlock(v_deform_fn, "find_startblock"); + LLVMAppendBasicBlockInContext(lc, v_deform_fn, "find_startblock"); b_out = - LLVMAppendBasicBlock(v_deform_fn, "outblock"); + LLVMAppendBasicBlockInContext(lc, v_deform_fn, "outblock"); b_dead = - LLVMAppendBasicBlock(v_deform_fn, "deadblock"); + LLVMAppendBasicBlockInContext(lc, v_deform_fn, "deadblock"); - b = LLVMCreateBuilder(); + b = LLVMCreateBuilderInContext(lc); attcheckattnoblocks = palloc(sizeof(LLVMBasicBlockRef) * natts); attstartblocks = palloc(sizeof(LLVMBasicBlockRef) * natts); @@ -249,7 +251,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_BITS, ""), - l_ptr(LLVMInt8Type()), + l_ptr(LLVMInt8TypeInContext(lc)), "t_bits"); v_infomask1 = l_load_struct_gep(b, @@ -267,14 +269,14 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, v_hasnulls = LLVMBuildICmp(b, LLVMIntNE, LLVMBuildAnd(b, - l_int16_const(HEAP_HASNULL), + l_int16_const(lc, HEAP_HASNULL), v_infomask1, ""), - l_int16_const(0), + l_int16_const(lc, 0), "hasnulls"); /* t_infomask2 & HEAP_NATTS_MASK */ v_maxatt = LLVMBuildAnd(b, - l_int16_const(HEAP_NATTS_MASK), + l_int16_const(lc, HEAP_NATTS_MASK), v_infomask2, "maxatt"); @@ -289,13 +291,13 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_HOFF, ""), - LLVMInt32Type(), "t_hoff"); + LLVMInt32TypeInContext(lc), "t_hoff"); v_tupdata_base = l_gep(b, - LLVMInt8Type(), + LLVMInt8TypeInContext(lc), LLVMBuildBitCast(b, v_tuplep, - l_ptr(LLVMInt8Type()), + l_ptr(LLVMInt8TypeInContext(lc)), ""), &v_hoff, 1, "v_tupdata_base"); @@ -307,7 +309,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, { LLVMValueRef v_off_start; - v_off_start = l_load(b, LLVMInt32Type(), v_slotoffp, "v_slot_off"); + v_off_start = l_load(b, LLVMInt32TypeInContext(lc), v_slotoffp, "v_slot_off"); v_off_start = LLVMBuildZExt(b, v_off_start, TypeSizeT, ""); LLVMBuildStore(b, v_off_start, v_offp); } @@ -353,7 +355,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntULT, v_maxatt, - l_int16_const(natts), + l_int16_const(lc, natts), ""), b_adjust_unavail_cols, b_find_start); @@ -362,8 +364,8 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols); v_params[0] = v_slot; - v_params[1] = LLVMBuildZExt(b, v_maxatt, LLVMInt32Type(), ""); - v_params[2] = l_int32_const(natts); + v_params[1] = LLVMBuildZExt(b, v_maxatt, LLVMInt32TypeInContext(lc), ""); + v_params[2] = l_int32_const(lc, natts); f = llvm_pg_func(mod, "slot_getmissingattrs"); l_call(b, LLVMGetFunctionType(f), f, @@ -373,7 +375,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, LLVMPositionBuilderAtEnd(b, b_find_start); - v_nvalid = l_load(b, LLVMInt16Type(), v_nvalidp, ""); + v_nvalid = l_load(b, LLVMInt16TypeInContext(lc), v_nvalidp, ""); /* * Build switch to go from nvalid to the right startblock. Callers @@ -388,7 +390,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, for (attnum = 0; attnum < natts; attnum++) { - LLVMValueRef v_attno = l_int16_const(attnum); + LLVMValueRef v_attno = l_int16_const(lc, attnum); LLVMAddCase(v_switch, v_attno, attcheckattnoblocks[attnum]); } @@ -412,7 +414,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, Form_pg_attribute att = TupleDescAttr(desc, attnum); LLVMValueRef v_incby; int alignto; - LLVMValueRef l_attno = l_int16_const(attnum); + LLVMValueRef l_attno = l_int16_const(lc, attnum); LLVMValueRef v_attdatap; LLVMValueRef v_resultp; @@ -473,14 +475,14 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, else b_next = attcheckattnoblocks[attnum + 1]; - v_nullbyteno = l_int32_const(attnum >> 3); - v_nullbytemask = l_int8_const(1 << ((attnum) & 0x07)); - v_nullbyte = l_load_gep1(b, LLVMInt8Type(), v_bits, v_nullbyteno, "attnullbyte"); + v_nullbyteno = l_int32_const(lc, attnum >> 3); + v_nullbytemask = l_int8_const(lc, 1 << ((attnum) & 0x07)); + v_nullbyte = l_load_gep1(b, LLVMInt8TypeInContext(lc), v_bits, v_nullbyteno, "attnullbyte"); v_nullbit = LLVMBuildICmp(b, LLVMIntEQ, LLVMBuildAnd(b, v_nullbyte, v_nullbytemask, ""), - l_int8_const(0), + l_int8_const(lc, 0), "attisnull"); v_attisnull = LLVMBuildAnd(b, v_hasnulls, v_nullbit, ""); @@ -491,8 +493,8 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, /* store null-byte */ LLVMBuildStore(b, - l_int8_const(1), - l_gep(b, LLVMInt8Type(), v_tts_nulls, &l_attno, 1, "")); + l_int8_const(lc, 1), + l_gep(b, LLVMInt8TypeInContext(lc), v_tts_nulls, &l_attno, 1, "")); /* store zero datum */ LLVMBuildStore(b, l_sizet_const(0), @@ -558,10 +560,11 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, v_off = l_load(b, TypeSizeT, v_offp, ""); v_possible_padbyte = - l_load_gep1(b, LLVMInt8Type(), v_tupdata_base, v_off, "padbyte"); + l_load_gep1(b, LLVMInt8TypeInContext(lc), v_tupdata_base, + v_off, "padbyte"); v_ispad = LLVMBuildICmp(b, LLVMIntEQ, - v_possible_padbyte, l_int8_const(0), + v_possible_padbyte, l_int8_const(lc, 0), "ispadbyte"); LLVMBuildCondBr(b, v_ispad, attalignblocks[attnum], @@ -669,14 +672,14 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); v_attdatap = - l_gep(b, LLVMInt8Type(), v_tupdata_base, &v_off, 1, ""); + l_gep(b, LLVMInt8TypeInContext(lc), v_tupdata_base, &v_off, 1, ""); } /* compute address to store value at */ v_resultp = l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, ""); /* store null-byte (false) */ - LLVMBuildStore(b, l_int8_const(0), + LLVMBuildStore(b, l_int8_const(lc, 0), l_gep(b, TypeStorageBool, v_tts_nulls, &l_attno, 1, "")); /* @@ -686,7 +689,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, if (att->attbyval) { LLVMValueRef v_tmp_loaddata; - LLVMTypeRef vartype = LLVMIntType(att->attlen * 8); + LLVMTypeRef vartype = LLVMIntTypeInContext(lc, att->attlen * 8); LLVMTypeRef vartypep = LLVMPointerType(vartype, 0); v_tmp_loaddata = @@ -778,11 +781,11 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); LLVMValueRef v_flags; - LLVMBuildStore(b, l_int16_const(natts), v_nvalidp); - v_off = LLVMBuildTrunc(b, v_off, LLVMInt32Type(), ""); + LLVMBuildStore(b, l_int16_const(lc, natts), v_nvalidp); + v_off = LLVMBuildTrunc(b, v_off, LLVMInt32TypeInContext(lc), ""); LLVMBuildStore(b, v_off, v_slotoffp); - v_flags = l_load(b, LLVMInt16Type(), v_flagsp, "tts_flags"); - v_flags = LLVMBuildOr(b, v_flags, l_int16_const(TTS_FLAG_SLOW), ""); + v_flags = l_load(b, LLVMInt16TypeInContext(lc), v_flagsp, "tts_flags"); + v_flags = LLVMBuildOr(b, v_flags, l_int16_const(lc, TTS_FLAG_SLOW), ""); LLVMBuildStore(b, v_flags, v_flagsp); LLVMBuildRetVoid(b); } diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c index f69780a5d7..c25960a82b 100644 --- a/src/backend/jit/llvm/llvmjit_expr.c +++ b/src/backend/jit/llvm/llvmjit_expr.c @@ -80,6 +80,7 @@ llvm_compile_expr(ExprState *state) LLVMBuilderRef b; LLVMModuleRef mod; + LLVMContextRef lc; LLVMValueRef eval_fn; LLVMBasicBlockRef entry; LLVMBasicBlockRef *opblocks; @@ -139,8 +140,9 @@ llvm_compile_expr(ExprState *state) INSTR_TIME_SET_CURRENT(starttime); mod = llvm_mutable_module(context); + lc = LLVMGetModuleContext(mod); - b = LLVMCreateBuilder(); + b = LLVMCreateBuilderInContext(lc); funcname = llvm_expand_funcname(context, "evalexpr"); @@ -151,7 +153,7 @@ llvm_compile_expr(ExprState *state) LLVMSetVisibility(eval_fn, LLVMDefaultVisibility); llvm_copy_attributes(AttributeTemplate, eval_fn); - entry = LLVMAppendBasicBlock(eval_fn, "entry"); + entry = LLVMAppendBasicBlockInContext(lc, eval_fn, "entry"); /* build state */ v_state = LLVMGetParam(eval_fn, 0); @@ -328,7 +330,7 @@ llvm_compile_expr(ExprState *state) ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntUGE, v_nvalid, - l_int16_const(op->d.fetch.last_var), + l_int16_const(lc, op->d.fetch.last_var), ""), opblocks[i + 1], b_fetch); @@ -364,7 +366,7 @@ llvm_compile_expr(ExprState *state) LLVMValueRef params[2]; params[0] = v_slot; - params[1] = l_int32_const(op->d.fetch.last_var); + params[1] = l_int32_const(lc, op->d.fetch.last_var); l_call(b, llvm_pg_var_func_type("slot_getsomeattrs_int"), @@ -402,7 +404,7 @@ llvm_compile_expr(ExprState *state) v_nulls = v_scannulls; } - v_attnum = l_int32_const(op->d.var.attnum); + v_attnum = l_int32_const(lc, op->d.var.attnum); value = l_load_gep1(b, TypeSizeT, v_values, v_attnum, ""); isnull = l_load_gep1(b, TypeStorageBool, v_nulls, v_attnum, ""); LLVMBuildStore(b, value, v_resvaluep); @@ -476,12 +478,12 @@ llvm_compile_expr(ExprState *state) } /* load data */ - v_attnum = l_int32_const(op->d.assign_var.attnum); + v_attnum = l_int32_const(lc, op->d.assign_var.attnum); v_value = l_load_gep1(b, TypeSizeT, v_values, v_attnum, ""); v_isnull = l_load_gep1(b, TypeStorageBool, v_nulls, v_attnum, ""); /* compute addresses of targets */ - v_resultnum = l_int32_const(op->d.assign_var.resultnum); + v_resultnum = l_int32_const(lc, op->d.assign_var.resultnum); v_rvaluep = l_gep(b, TypeSizeT, v_resultvalues, @@ -513,7 +515,7 @@ llvm_compile_expr(ExprState *state) v_isnull = l_load(b, TypeStorageBool, v_tmpisnullp, ""); /* compute addresses of targets */ - v_resultnum = l_int32_const(resultnum); + v_resultnum = l_int32_const(lc, resultnum); v_rvaluep = l_gep(b, TypeSizeT, v_resultvalues, &v_resultnum, 1, ""); v_risnullp = @@ -547,7 +549,7 @@ llvm_compile_expr(ExprState *state) v_isnull = l_load(b, TypeStorageBool, v_tmpisnullp, ""); /* compute addresses of targets */ - v_resultnum = l_int32_const(resultnum); + v_resultnum = l_int32_const(lc, resultnum); v_rvaluep = l_gep(b, TypeSizeT, v_resultvalues, &v_resultnum, 1, ""); v_risnullp = @@ -1746,7 +1748,7 @@ llvm_compile_expr(ExprState *state) v_cmpresult = LLVMBuildTrunc(b, l_load(b, TypeSizeT, v_resvaluep, ""), - LLVMInt32Type(), ""); + LLVMInt32TypeInContext(lc), ""); switch (rctype) { @@ -1772,7 +1774,7 @@ llvm_compile_expr(ExprState *state) v_result = LLVMBuildICmp(b, predicate, v_cmpresult, - l_int32_const(0), + l_int32_const(lc, 0), ""); v_result = LLVMBuildZExt(b, v_result, TypeSizeT, ""); @@ -1931,8 +1933,8 @@ llvm_compile_expr(ExprState *state) * load it from memory each time round. */ v_aggnop = l_ptr_const(&aggref->aggno, - l_ptr(LLVMInt32Type())); - v_aggno = l_load(b, LLVMInt32Type(), v_aggnop, "v_aggno"); + l_ptr(LLVMInt32TypeInContext(lc))); + v_aggno = l_load(b, LLVMInt32TypeInContext(lc), v_aggnop, "v_aggno"); /* load agg value / null */ value = l_load_gep1(b, TypeSizeT, v_aggvalues, v_aggno, "aggvalue"); @@ -1966,8 +1968,8 @@ llvm_compile_expr(ExprState *state) * expression). So load it from memory each time round. */ v_wfuncnop = l_ptr_const(&wfunc->wfuncno, - l_ptr(LLVMInt32Type())); - v_wfuncno = l_load(b, LLVMInt32Type(), v_wfuncnop, "v_wfuncno"); + l_ptr(LLVMInt32TypeInContext(lc))); + v_wfuncno = l_load(b, LLVMInt32TypeInContext(lc), v_wfuncnop, "v_wfuncno"); /* load window func value / null */ value = l_load_gep1(b, TypeSizeT, v_aggvalues, v_wfuncno, @@ -2082,7 +2084,7 @@ llvm_compile_expr(ExprState *state) /* strict function, check for NULL args */ for (argno = 0; argno < nargs; argno++) { - LLVMValueRef v_argno = l_int32_const(argno); + LLVMValueRef v_argno = l_int32_const(lc, argno); LLVMValueRef v_argisnull; LLVMBasicBlockRef b_argnotnull; @@ -2156,8 +2158,8 @@ llvm_compile_expr(ExprState *state) v_aggstatep, FIELDNO_AGGSTATE_ALL_PERGROUPS, "aggstate.all_pergroups"); - v_setoff = l_int32_const(op->d.agg_init_trans.setoff); - v_transno = l_int32_const(op->d.agg_init_trans.transno); + v_setoff = l_int32_const(lc, op->d.agg_init_trans.setoff); + v_transno = l_int32_const(lc, op->d.agg_init_trans.transno); v_pergroupp = l_gep(b, StructAggStatePerGroupData, @@ -2205,7 +2207,7 @@ llvm_compile_expr(ExprState *state) FIELDNO_AGGSTATE_CURAGGCONTEXT, "aggstate.curaggcontext"); - LLVMBuildStore(b, l_int32_const(op->d.agg_init_trans.setno), + LLVMBuildStore(b, l_int32_const(lc, op->d.agg_init_trans.setno), v_current_set); LLVMBuildStore(b, v_aggcontext, v_curaggcontext); @@ -2254,9 +2256,9 @@ llvm_compile_expr(ExprState *state) FIELDNO_AGGSTATE_ALL_PERGROUPS, "aggstate.all_pergroups"); v_setoff = - l_int32_const(op->d.agg_strict_trans_check.setoff); + l_int32_const(lc, op->d.agg_strict_trans_check.setoff); v_transno = - l_int32_const(op->d.agg_strict_trans_check.transno); + l_int32_const(lc, op->d.agg_strict_trans_check.transno); v_pergroupp = l_gep(b, StructAggStatePerGroupData, @@ -2334,8 +2336,8 @@ llvm_compile_expr(ExprState *state) v_aggstatep, FIELDNO_AGGSTATE_ALL_PERGROUPS, "aggstate.all_pergroups"); - v_setoff = l_int32_const(op->d.agg_trans.setoff); - v_transno = l_int32_const(op->d.agg_trans.transno); + v_setoff = l_int32_const(lc, op->d.agg_trans.setoff); + v_transno = l_int32_const(lc, op->d.agg_trans.transno); v_pergroupp = l_gep(b, StructAggStatePerGroupData, @@ -2369,7 +2371,7 @@ llvm_compile_expr(ExprState *state) /* set aggstate globals */ LLVMBuildStore(b, v_aggcontext, v_curaggcontext); - LLVMBuildStore(b, l_int32_const(op->d.agg_trans.setno), + LLVMBuildStore(b, l_int32_const(lc, op->d.agg_trans.setno), v_current_setp); LLVMBuildStore(b, v_pertransp, v_current_pertransp); @@ -2563,11 +2565,14 @@ BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b, LLVMModuleRef mod, FunctionCallInfo fcinfo, LLVMValueRef *v_fcinfo_isnull) { + LLVMContextRef lc; LLVMValueRef v_fn; LLVMValueRef v_fcinfo_isnullp; LLVMValueRef v_retval; LLVMValueRef v_fcinfo; + lc = LLVMGetModuleContext(mod); + v_fn = llvm_function_reference(context, b, mod, fcinfo); v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData)); @@ -2591,12 +2596,12 @@ BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b, LLVMValueRef v_lifetime = create_LifetimeEnd(mod); LLVMValueRef params[2]; - params[0] = l_int64_const(sizeof(NullableDatum) * fcinfo->nargs); - params[1] = l_ptr_const(fcinfo->args, l_ptr(LLVMInt8Type())); + params[0] = l_int64_const(lc, sizeof(NullableDatum) * fcinfo->nargs); + params[1] = l_ptr_const(fcinfo->args, l_ptr(LLVMInt8TypeInContext(lc))); l_call(b, LLVMGetFunctionType(v_lifetime), v_lifetime, params, lengthof(params), ""); - params[0] = l_int64_const(sizeof(fcinfo->isnull)); - params[1] = l_ptr_const(&fcinfo->isnull, l_ptr(LLVMInt8Type())); + params[0] = l_int64_const(lc, sizeof(fcinfo->isnull)); + params[1] = l_ptr_const(&fcinfo->isnull, l_ptr(LLVMInt8TypeInContext(lc))); l_call(b, LLVMGetFunctionType(v_lifetime), v_lifetime, params, lengthof(params), ""); } @@ -2642,6 +2647,7 @@ create_LifetimeEnd(LLVMModuleRef mod) LLVMTypeRef sig; LLVMValueRef fn; LLVMTypeRef param_types[2]; + LLVMContextRef lc; /* LLVM 5+ has a variadic pointer argument */ #if LLVM_VERSION_MAJOR < 5 @@ -2654,12 +2660,12 @@ create_LifetimeEnd(LLVMModuleRef mod) if (fn) return fn; - param_types[0] = LLVMInt64Type(); - param_types[1] = l_ptr(LLVMInt8Type()); + lc = LLVMGetModuleContext(mod); + param_types[0] = LLVMInt64TypeInContext(lc); + param_types[1] = l_ptr(LLVMInt8TypeInContext(lc)); - sig = LLVMFunctionType(LLVMVoidType(), - param_types, lengthof(param_types), - false); + sig = LLVMFunctionType(LLVMVoidTypeInContext(lc), param_types, + lengthof(param_types), false); fn = LLVMAddFunction(mod, nm, sig); LLVMSetFunctionCallConv(fn, LLVMCCallConv); diff --git a/src/backend/jit/llvm/llvmjit_inline.cpp b/src/backend/jit/llvm/llvmjit_inline.cpp index 123c4330e2..295375e237 100644 --- a/src/backend/jit/llvm/llvmjit_inline.cpp +++ b/src/backend/jit/llvm/llvmjit_inline.cpp @@ -114,12 +114,12 @@ typedef llvm::StringMap > SummaryCache llvm::ManagedStatic summary_cache; -static std::unique_ptr llvm_build_inline_plan(llvm::Module *mod); +static std::unique_ptr llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod); static void llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline); -static llvm::Module* load_module_cached(llvm::StringRef modPath); -static std::unique_ptr load_module(llvm::StringRef Identifier); +static llvm::Module* load_module_cached(LLVMContextRef c, llvm::StringRef modPath); +static std::unique_ptr load_module(LLVMContextRef c, llvm::StringRef Identifier); static std::unique_ptr llvm_load_summary(llvm::StringRef path); @@ -152,6 +152,18 @@ summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid); #define ilog(...) (void) 0 #endif +/* + * Reset inlining related state. This needs to be called before the currently + * used LLVMContextRef is disposed (and a new one create), otherwise we would + * have dangling references to deleted modules. + */ +void +llvm_inline_reset_caches(void) +{ + module_cache->clear(); + summary_cache->clear(); +} + /* * Perform inlining of external function references in M based on a simple * cost based analysis. @@ -159,9 +171,10 @@ summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid); void llvm_inline(LLVMModuleRef M) { + LLVMContextRef lc = LLVMGetModuleContext(M); llvm::Module *mod = llvm::unwrap(M); - std::unique_ptr globalsToInline = llvm_build_inline_plan(mod); + std::unique_ptr globalsToInline = llvm_build_inline_plan(lc, mod); if (!globalsToInline) return; llvm_execute_inline_plan(mod, globalsToInline.get()); @@ -172,7 +185,7 @@ llvm_inline(LLVMModuleRef M) * mod. */ static std::unique_ptr -llvm_build_inline_plan(llvm::Module *mod) +llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod) { std::unique_ptr globalsToInline(new ImportMapTy()); FunctionInlineStates functionStates; @@ -271,7 +284,7 @@ llvm_build_inline_plan(llvm::Module *mod) continue; } - defMod = load_module_cached(modPath); + defMod = load_module_cached(lc, modPath); if (defMod->materializeMetadata()) elog(FATAL, "failed to materialize metadata"); @@ -466,20 +479,20 @@ llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline) * the cache state would get corrupted. */ static llvm::Module* -load_module_cached(llvm::StringRef modPath) +load_module_cached(LLVMContextRef lc, llvm::StringRef modPath) { auto it = module_cache->find(modPath); if (it == module_cache->end()) { it = module_cache->insert( - std::make_pair(modPath, load_module(modPath))).first; + std::make_pair(modPath, load_module(lc, modPath))).first; } return it->second.get(); } static std::unique_ptr -load_module(llvm::StringRef Identifier) +load_module(LLVMContextRef lc, llvm::StringRef Identifier) { LLVMMemoryBufferRef buf; LLVMModuleRef mod; @@ -491,7 +504,7 @@ load_module(llvm::StringRef Identifier) if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg)) elog(FATAL, "failed to open bitcode file \"%s\": %s", path, msg); - if (LLVMGetBitcodeModuleInContext2(LLVMGetGlobalContext(), buf, &mod)) + if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod)) elog(FATAL, "failed to parse bitcode in file \"%s\"", path); /* diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h index 4009b43055..dff3fd5327 100644 --- a/src/include/jit/llvmjit.h +++ b/src/include/jit/llvmjit.h @@ -44,6 +44,13 @@ typedef struct LLVMJitContext /* number of modules created */ size_t module_generation; + /* + * The LLVM Context used by this JIT context. An LLVM context is reused + * across many compilations, but occasionally reset to prevent it using + * too much memory due to more and more types accumulating. + */ + LLVMContextRef llvm_context; + /* current, "open for write", module */ LLVMModuleRef module; @@ -107,6 +114,7 @@ extern LLVMValueRef llvm_function_reference(LLVMJitContext *context, LLVMModuleRef mod, FunctionCallInfo fcinfo); +extern void llvm_inline_reset_caches(void); extern void llvm_inline(LLVMModuleRef mod); /* diff --git a/src/include/jit/llvmjit_emit.h b/src/include/jit/llvmjit_emit.h index 0d06383347..36659e201b 100644 --- a/src/include/jit/llvmjit_emit.h +++ b/src/include/jit/llvmjit_emit.h @@ -46,36 +46,36 @@ l_ptr(LLVMTypeRef t) * Emit constant integer. */ static inline LLVMValueRef -l_int8_const(int8 i) +l_int8_const(LLVMContextRef lc, int8 i) { - return LLVMConstInt(LLVMInt8Type(), i, false); + return LLVMConstInt(LLVMInt8TypeInContext(lc), i, false); } /* * Emit constant integer. */ static inline LLVMValueRef -l_int16_const(int16 i) +l_int16_const(LLVMContextRef lc, int16 i) { - return LLVMConstInt(LLVMInt16Type(), i, false); + return LLVMConstInt(LLVMInt16TypeInContext(lc), i, false); } /* * Emit constant integer. */ static inline LLVMValueRef -l_int32_const(int32 i) +l_int32_const(LLVMContextRef lc, int32 i) { - return LLVMConstInt(LLVMInt32Type(), i, false); + return LLVMConstInt(LLVMInt32TypeInContext(lc), i, false); } /* * Emit constant integer. */ static inline LLVMValueRef -l_int64_const(int64 i) +l_int64_const(LLVMContextRef lc, int64 i) { - return LLVMConstInt(LLVMInt64Type(), i, false); + return LLVMConstInt(LLVMInt64TypeInContext(lc), i, false); } /* @@ -178,12 +178,15 @@ l_bb_before_v(LLVMBasicBlockRef r, const char *fmt,...) { char buf[512]; va_list args; + LLVMContextRef lc; va_start(args, fmt); vsnprintf(buf, sizeof(buf), fmt, args); va_end(args); - return LLVMInsertBasicBlock(r, buf); + lc = LLVMGetTypeContext(LLVMTypeOf(LLVMGetBasicBlockParent(r))); + + return LLVMInsertBasicBlockInContext(lc, r, buf); } /* separate, because pg_attribute_printf(2, 3) can't appear in definition */ @@ -198,12 +201,15 @@ l_bb_append_v(LLVMValueRef f, const char *fmt,...) { char buf[512]; va_list args; + LLVMContextRef lc; va_start(args, fmt); vsnprintf(buf, sizeof(buf), fmt, args); va_end(args); - return LLVMAppendBasicBlock(f, buf); + lc = LLVMGetTypeContext(LLVMTypeOf(f)); + + return LLVMAppendBasicBlockInContext(lc, f, buf); } /* @@ -215,7 +221,7 @@ l_callsite_ro(LLVMValueRef f) const char argname[] = "readonly"; LLVMAttributeRef ref; - ref = LLVMCreateStringAttribute(LLVMGetGlobalContext(), + ref = LLVMCreateStringAttribute(LLVMGetTypeContext(LLVMTypeOf(f)), argname, sizeof(argname) - 1, NULL, 0); @@ -235,7 +241,7 @@ l_callsite_alwaysinline(LLVMValueRef f) id = LLVMGetEnumAttributeKindForName(argname, sizeof(argname) - 1); - attr = LLVMCreateEnumAttribute(LLVMGetGlobalContext(), id, 0); + attr = LLVMCreateEnumAttribute(LLVMGetTypeContext(LLVMTypeOf(f)), id, 0); LLVMAddCallSiteAttribute(f, LLVMAttributeFunctionIndex, attr); }