rdelaage/postgresql
1
0
Fork 0
mirror of https://git.postgresql.org/git/postgresql.git synced 2024-09-28 00:11:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

Optimize GenerationAlloc() and SlabAlloc()

Browse Source 
In a similar effort to 413c18401, separate out the hot and cold paths in
GenerationAlloc() and SlabAlloc() to avoid having to setup the stack frame
for the hot path.

This additionally adjusts how we use the GenerationContext's freeblock.
Freeblock, when set, is now always empty and we only switch to using it
when the current allocation request finds the current block does not have
enough space and the freeblock is large enough to accomodate the
allocation.

This commit also adjusts GenerationFree() so that if we pfree the final
allocation in the current generation block, we now mark that block as
empty and keep it as the current block.  Previously we free'd that block
and set the current block to NULL.  Doing that meant we needed a special
case in GenerationAlloc to check if GenerationContext.block was NULL.
So this both reduces free/malloc calls and reduces the work done in
GenerationAlloc().

In passing, improve some comments in aset.c

Discussion: https://postgr.es/m/CAApHDvpHVSJqqb4B4OZLixr=CotKq-eKkbwZqvZVo_biYvUvQA@mail.gmail.com
This commit is contained in:
David Rowley 2024-03-04 17:42:10 +13:00
parent 07c36c1333
commit a0cd954480
3 changed files with 386 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/backend/utils/mmgr/aset.c
View File

@ -943,8 +943,9 @@ AllocSetAllocFromNewBlock(MemoryContext context, Size size, int flags,
/*
* AllocSetAlloc
* Returns pointer to allocated memory of given size or NULL if
* request could not be completed; memory is added to the set.
* Returns a pointer to allocated memory of given size or raises an ERROR
* on allocation failure, or returns NULL when flags contains
* MCXT_ALLOC_NO_OOM.
*
* No request may exceed:
* MAXALIGN_DOWN(SIZE_MAX) - ALLOC_BLOCKHDRSZ - ALLOC_CHUNKHDRSZ
@ -955,11 +956,12 @@ AllocSetAllocFromNewBlock(MemoryContext context, Size size, int flags,
* return space that is marked NOACCESS - AllocSetRealloc has to beware!
*
* This function should only contain the most common code paths. Everything
* else should be in pg_noinline helper functions, thus avoiding the overheads
* creating a stack frame for the common cases. Allocating memory is often a
* bottleneck in many workloads, so avoiding stack frame setup is worthwhile.
* Helper functions should always directly return the newly allocated memory
* so that we can just return that address directly as a tail call.
* else should be in pg_noinline helper functions, thus avoiding the overhead
* of creating a stack frame for the common cases. Allocating memory is often
* a bottleneck in many workloads, so avoiding stack frame setup is
* worthwhile. Helper functions should always directly return the newly
* allocated memory so that we can just return that address directly as a tail
* call.
*/
void *
AllocSetAlloc(MemoryContext context, Size size, int flags)

415
src/backend/utils/mmgr/generation.c
View File

@ -66,11 +66,9 @@ typedef struct GenerationContext
uint32 nextBlockSize; /* next block size to allocate */
uint32 allocChunkLimit; /* effective chunk size limit */
GenerationBlock *block; /* current (most recently allocated) block, or
* NULL if we've just freed the most recent
* block */
GenerationBlock *freeblock; /* pointer to a block that's being recycled,
* or NULL if there's no such block. */
GenerationBlock *block; /* current (most recently allocated) block */
GenerationBlock *freeblock; /* pointer to an empty block that's being
* recycled, or NULL if there's no such block. */
dlist_head blocks; /* list of blocks */
} GenerationContext;
@ -111,6 +109,12 @@ struct GenerationBlock
#define GenerationBlockIsValid(block) \
(PointerIsValid(block) && GenerationIsValid((block)->context))
/*
* GenerationBlockIsEmpty
* True iff block contains no chunks
*/
#define GenerationBlockIsEmpty(b) ((b)->nchunks == 0)
/*
* We always store external chunks on a dedicated block. This makes fetching
* the block from an external chunk easy since it's always the first and only
@ -131,7 +135,6 @@ struct GenerationBlock
static inline void GenerationBlockInit(GenerationContext *context,
GenerationBlock *block,
Size blksize);
static inline bool GenerationBlockIsEmpty(GenerationBlock *block);
static inline void GenerationBlockMarkEmpty(GenerationBlock *block);
static inline Size GenerationBlockFreeBytes(GenerationBlock *block);
static inline void GenerationBlockFree(GenerationContext *set,
@ -331,28 +334,23 @@ GenerationDelete(MemoryContext context)
}
/*
* GenerationAlloc
* Returns pointer to allocated memory of given size or NULL if
* request could not be completed; memory is added to the set.
* Helper for GenerationAlloc() that allocates an entire block for the chunk.
*
* No request may exceed:
* MAXALIGN_DOWN(SIZE_MAX) - Generation_BLOCKHDRSZ - Generation_CHUNKHDRSZ
* All callers use a much-lower limit.
*
* Note: when using valgrind, it doesn't matter how the returned allocation
* is marked, as mcxt.c will set it to UNDEFINED. In some paths we will
* return space that is marked NOACCESS - GenerationRealloc has to beware!
* GenerationAlloc()'s comment explains why this is separate.
*/
void *
GenerationAlloc(MemoryContext context, Size size, int flags)
pg_noinline
static void *
GenerationAllocLarge(MemoryContext context, Size size, int flags)
{
GenerationContext *set = (GenerationContext *) context;
GenerationBlock *block;
MemoryChunk *chunk;
Size chunk_size;
Size required_size;
Size blksize;
Assert(GenerationIsValid(set));
/* validate 'size' is within the limits for the given 'flags' */
MemoryContextCheckSize(context, size, flags);
#ifdef MEMORY_CONTEXT_CHECKING
/* ensure there's always space for the sentinel byte */
@ -361,143 +359,66 @@ GenerationAlloc(MemoryContext context, Size size, int flags)
chunk_size = MAXALIGN(size);
#endif
required_size = chunk_size + Generation_CHUNKHDRSZ;
blksize = required_size + Generation_BLOCKHDRSZ;
/* is it an over-sized chunk? if yes, allocate special block */
if (chunk_size > set->allocChunkLimit)
{
Size blksize = required_size + Generation_BLOCKHDRSZ;
block = (GenerationBlock *) malloc(blksize);
if (block == NULL)
return MemoryContextAllocationFailure(context, size, flags);
/* only check size in paths where the limits could be hit */
MemoryContextCheckSize((MemoryContext) set, size, flags);
context->mem_allocated += blksize;
block = (GenerationBlock *) malloc(blksize);
if (block == NULL)
return MemoryContextAllocationFailure(context, size, flags);
/* block with a single (used) chunk */
block->context = set;
block->blksize = blksize;
block->nchunks = 1;
block->nfree = 0;
context->mem_allocated += blksize;
/* the block is completely full */
block->freeptr = block->endptr = ((char *) block) + blksize;
/* block with a single (used) chunk */
block->context = set;
block->blksize = blksize;
block->nchunks = 1;
block->nfree = 0;
chunk = (MemoryChunk *) (((char *) block) + Generation_BLOCKHDRSZ);
/* the block is completely full */
block->freeptr = block->endptr = ((char *) block) + blksize;
chunk = (MemoryChunk *) (((char *) block) + Generation_BLOCKHDRSZ);
/* mark the MemoryChunk as externally managed */
MemoryChunkSetHdrMaskExternal(chunk, MCTX_GENERATION_ID);
/* mark the MemoryChunk as externally managed */
MemoryChunkSetHdrMaskExternal(chunk, MCTX_GENERATION_ID);
#ifdef MEMORY_CONTEXT_CHECKING
chunk->requested_size = size;
/* set mark to catch clobber of "unused" space */
Assert(size < chunk_size);
set_sentinel(MemoryChunkGetPointer(chunk), size);
chunk->requested_size = size;
/* set mark to catch clobber of "unused" space */
Assert(size < chunk_size);
set_sentinel(MemoryChunkGetPointer(chunk), size);
#endif
#ifdef RANDOMIZE_ALLOCATED_MEMORY
/* fill the allocated space with junk */
randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
/* fill the allocated space with junk */
randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
#endif
/* add the block to the list of allocated blocks */
dlist_push_head(&set->blocks, &block->node);
/* add the block to the list of allocated blocks */
dlist_push_head(&set->blocks, &block->node);
/* Ensure any padding bytes are marked NOACCESS. */
VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
chunk_size - size);
/* Ensure any padding bytes are marked NOACCESS. */
VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
chunk_size - size);
/* Disallow access to the chunk header. */
VALGRIND_MAKE_MEM_NOACCESS(chunk, Generation_CHUNKHDRSZ);
/* Disallow access to the chunk header. */
VALGRIND_MAKE_MEM_NOACCESS(chunk, Generation_CHUNKHDRSZ);
return MemoryChunkGetPointer(chunk);
}
return MemoryChunkGetPointer(chunk);
}
/*
* Not an oversized chunk. We try to first make use of the current block,
* but if there's not enough space in it, instead of allocating a new
* block, we look to see if the freeblock is empty and has enough space.
* If not, we'll also try the same using the keeper block. The keeper
* block may have become empty and we have no other way to reuse it again
* if we don't try to use it explicitly here.
*
* We don't want to start filling the freeblock before the current block
* is full, otherwise we may cause fragmentation in FIFO type workloads.
* We only switch to using the freeblock or keeper block if those blocks
* are completely empty. If we didn't do that we could end up fragmenting
* consecutive allocations over multiple blocks which would be a problem
* that would compound over time.
*/
block = set->block;
/*
* Small helper for allocating a new chunk from a chunk, to avoid duplicating
* the code between GenerationAlloc() and GenerationAllocFromNewBlock().
*/
static inline void *
GenerationAllocChunkFromBlock(MemoryContext context, GenerationBlock *block,
Size size, Size chunk_size)
{
MemoryChunk *chunk = (MemoryChunk *) (block->freeptr);
if (block == NULL ||
GenerationBlockFreeBytes(block) < required_size)
{
Size blksize;
GenerationBlock *freeblock = set->freeblock;
if (freeblock != NULL &&
GenerationBlockIsEmpty(freeblock) &&
GenerationBlockFreeBytes(freeblock) >= required_size)
{
block = freeblock;
/*
* Zero out the freeblock as we'll set this to the current block
* below
*/
set->freeblock = NULL;
}
else if (GenerationBlockIsEmpty(KeeperBlock(set)) &&
GenerationBlockFreeBytes(KeeperBlock(set)) >= required_size)
{
block = KeeperBlock(set);
}
else
{
/*
* The first such block has size initBlockSize, and we double the
* space in each succeeding block, but not more than maxBlockSize.
*/
blksize = set->nextBlockSize;
set->nextBlockSize <<= 1;
if (set->nextBlockSize > set->maxBlockSize)
set->nextBlockSize = set->maxBlockSize;
/* we'll need a block hdr too, so add that to the required size */
required_size += Generation_BLOCKHDRSZ;
/* round the size up to the next power of 2 */
if (blksize < required_size)
blksize = pg_nextpower2_size_t(required_size);
block = (GenerationBlock *) malloc(blksize);
if (block == NULL)
return MemoryContextAllocationFailure(context, size, flags);
context->mem_allocated += blksize;
/* initialize the new block */
GenerationBlockInit(set, block, blksize);
/* add it to the doubly-linked list of blocks */
dlist_push_head(&set->blocks, &block->node);
/* Zero out the freeblock in case it's become full */
set->freeblock = NULL;
}
/* and also use it as the current allocation block */
set->block = block;
}
/* we're supposed to have a block with enough free space now */
/* validate we've been given a block with enough free space */
Assert(block != NULL);
Assert((block->endptr - block->freeptr) >= Generation_CHUNKHDRSZ + chunk_size);
chunk = (MemoryChunk *) block->freeptr;
Assert((block->endptr - block->freeptr) >=
Generation_CHUNKHDRSZ + chunk_size);
/* Prepare to initialize the chunk header. */
VALGRIND_MAKE_MEM_UNDEFINED(chunk, Generation_CHUNKHDRSZ);
@ -529,6 +450,156 @@ GenerationAlloc(MemoryContext context, Size size, int flags)
return MemoryChunkGetPointer(chunk);
}
/*
* Helper for GenerationAlloc() that allocates a new block and returns a chunk
* allocated from it.
*
* GenerationAlloc()'s comment explains why this is separate.
*/
pg_noinline
static void *
GenerationAllocFromNewBlock(MemoryContext context, Size size, int flags,
Size chunk_size)
{
GenerationContext *set = (GenerationContext *) context;
GenerationBlock *block;
Size blksize;
Size required_size;
/*
* The first such block has size initBlockSize, and we double the space in
* each succeeding block, but not more than maxBlockSize.
*/
blksize = set->nextBlockSize;
set->nextBlockSize <<= 1;
if (set->nextBlockSize > set->maxBlockSize)
set->nextBlockSize = set->maxBlockSize;
/* we'll need space for the chunk, chunk hdr and block hdr */
required_size = chunk_size + Generation_CHUNKHDRSZ + Generation_BLOCKHDRSZ;
/* round the size up to the next power of 2 */
if (blksize < required_size)
blksize = pg_nextpower2_size_t(required_size);
block = (GenerationBlock *) malloc(blksize);
if (block == NULL)
return MemoryContextAllocationFailure(context, size, flags);
context->mem_allocated += blksize;
/* initialize the new block */
GenerationBlockInit(set, block, blksize);
/* add it to the doubly-linked list of blocks */
dlist_push_head(&set->blocks, &block->node);
/* make this the current block */
set->block = block;
return GenerationAllocChunkFromBlock(context, block, size, chunk_size);
}
/*
* GenerationAlloc
* Returns a pointer to allocated memory of given size or raises an ERROR
* on allocation failure, or returns NULL when flags contains
* MCXT_ALLOC_NO_OOM.
*
* No request may exceed:
* MAXALIGN_DOWN(SIZE_MAX) - Generation_BLOCKHDRSZ - Generation_CHUNKHDRSZ
* All callers use a much-lower limit.
*
* Note: when using valgrind, it doesn't matter how the returned allocation
* is marked, as mcxt.c will set it to UNDEFINED. In some paths we will
* return space that is marked NOACCESS - GenerationRealloc has to beware!
*
* This function should only contain the most common code paths. Everything
* else should be in pg_noinline helper functions, thus avoiding the overhead
* of creating a stack frame for the common cases. Allocating memory is often
* a bottleneck in many workloads, so avoiding stack frame setup is
* worthwhile. Helper functions should always directly return the newly
* allocated memory so that we can just return that address directly as a tail
* call.
*/
void *
GenerationAlloc(MemoryContext context, Size size, int flags)
{
GenerationContext *set = (GenerationContext *) context;
GenerationBlock *block;
Size chunk_size;
Size required_size;
Assert(GenerationIsValid(set));
#ifdef MEMORY_CONTEXT_CHECKING
/* ensure there's always space for the sentinel byte */
chunk_size = MAXALIGN(size + 1);
#else
chunk_size = MAXALIGN(size);
#endif
/*
* If requested size exceeds maximum for chunks we hand the the request
* off to GenerationAllocLarge().
*/
if (chunk_size > set->allocChunkLimit)
return GenerationAllocLarge(context, size, flags);
required_size = chunk_size + Generation_CHUNKHDRSZ;
/*
* Not an oversized chunk. We try to first make use of the current block,
* but if there's not enough space in it, instead of allocating a new
* block, we look to see if the empty freeblock has enough space. We
* don't try reusing the keeper block. If it's become empty we'll reuse
* that again only if the context is reset.
*
* We only try reusing the freeblock if we've no space for this allocation
* on the current block. When a freeblock exists, we'll switch to it once
* the first time we can't fit an allocation in the current block. We
* avoid ping-ponging between the two as we need to be careful not to
* fragment differently sized consecutive allocations between several
* blocks. Going between the two could cause fragmentation for FIFO
* workloads, which generation is meant to be good at.
*/
block = set->block;
if (unlikely(GenerationBlockFreeBytes(block) < required_size))
{
GenerationBlock *freeblock = set->freeblock;
/* freeblock, if set, must be empty */
Assert(freeblock == NULL || GenerationBlockIsEmpty(freeblock));
/* check if we have a freeblock and if it's big enough */
if (freeblock != NULL &&
GenerationBlockFreeBytes(freeblock) >= required_size)
{
/* make the freeblock the current block */
set->freeblock = NULL;
set->block = freeblock;
return GenerationAllocChunkFromBlock(context,
freeblock,
size,
chunk_size);
}
else
{
/*
* No freeblock, or it's not big enough for this allocation. Make
* a new block.
*/
return GenerationAllocFromNewBlock(context, size, flags, chunk_size);
}
}
/* The current block has space, so just allocate chunk there. */
return GenerationAllocChunkFromBlock(context, block, size, chunk_size);
}
/*
* GenerationBlockInit
* Initializes 'block' assuming 'blksize'. Does not update the context's
@ -551,16 +622,6 @@ GenerationBlockInit(GenerationContext *context, GenerationBlock *block,
blksize - Generation_BLOCKHDRSZ);
}
/*
* GenerationBlockIsEmpty
* Returns true iff 'block' contains no chunks
*/
static inline bool
GenerationBlockIsEmpty(GenerationBlock *block)
{
return (block->nchunks == 0);
}
/*
* GenerationBlockMarkEmpty
* Set a block as empty. Does not free the block.
@ -621,8 +682,8 @@ GenerationBlockFree(GenerationContext *set, GenerationBlock *block)
/*
* GenerationFree
* Update number of chunks in the block, and if all chunks in the block
* are now free then discard the block.
* Update number of chunks in the block, and consider freeing the block
* if it's become empty.
*/
void
GenerationFree(void *pointer)
@ -692,45 +753,41 @@ GenerationFree(void *pointer)
Assert(block->nchunks > 0);
Assert(block->nfree <= block->nchunks);
Assert(block != block->context->freeblock);
/* If there are still allocated chunks in the block, we're done. */
if (block->nfree < block->nchunks)
if (likely(block->nfree < block->nchunks))
return;
set = block->context;
/* Don't try to free the keeper block, just mark it empty */
if (IsKeeperBlock(set, block))
{
GenerationBlockMarkEmpty(block);
return;
}
/*
* If there is no freeblock set or if this is the freeblock then instead
* of freeing this memory, we keep it around so that new allocations have
* the option of recycling it.
/*-----------------------
* The block this allocation was on has now become completely empty of
* chunks. In the general case, we can now return the memory for this
* block back to malloc. However, there are cases where we don't want to
* do that:
*
* 1) If it's the keeper block. This block was malloc'd in the same
* allocation as the context itself and can't be free'd without
* freeing the context.
* 2) If it's the current block. We could free this, but doing so would
* leave us nothing to set the current block to, so we just mark the
* block as empty so new allocations can reuse it again.
* 3) If we have no "freeblock" set, then we save a single block for
* future allocations to avoid having to malloc a new block again.
* This is useful for FIFO workloads as it avoids continual
* free/malloc cycles.
*/
if (set->freeblock == NULL || set->freeblock == block)
if (IsKeeperBlock(set, block) || set->block == block)
GenerationBlockMarkEmpty(block); /* case 1 and 2 */
else if (set->freeblock == NULL)
{
/* XXX should we only recycle maxBlockSize sized blocks? */
/* case 3 */
GenerationBlockMarkEmpty(block);
set->freeblock = block;
GenerationBlockMarkEmpty(block);
return;
}
/* Also make sure the block is not marked as the current block. */
if (set->block == block)
set->block = NULL;
/*
* The block is empty, so let's get rid of it. First remove it from the
* list of blocks, then return it to malloc().
*/
dlist_delete(&block->node);
set->header.mem_allocated -= block->blksize;
free(block);
else
GenerationBlockFree(set, block); /* Otherwise, free it */
}
/*

232
src/backend/utils/mmgr/slab.c
View File

@ -490,10 +490,141 @@ SlabDelete(MemoryContext context)
free(context);
}
/*
* Small helper for allocating a new chunk from a chunk, to avoid duplicating
* the code between SlabAlloc() and SlabAllocFromNewBlock().
*/
static inline void *
SlabAllocSetupNewChunk(MemoryContext context, SlabBlock *block,
MemoryChunk *chunk, Size size)
{
SlabContext *slab = (SlabContext *) context;
/*
* Check that the chunk pointer is actually somewhere on the block and is
* aligned as expected.
*/
Assert(chunk >= SlabBlockGetChunk(slab, block, 0));
Assert(chunk <= SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1));
Assert(SlabChunkMod(slab, block, chunk) == 0);
/* Prepare to initialize the chunk header. */
VALGRIND_MAKE_MEM_UNDEFINED(chunk, Slab_CHUNKHDRSZ);
MemoryChunkSetHdrMask(chunk, block, MAXALIGN(slab->chunkSize), MCTX_SLAB_ID);
#ifdef MEMORY_CONTEXT_CHECKING
/* slab mark to catch clobber of "unused" space */
Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
set_sentinel(MemoryChunkGetPointer(chunk), size);
VALGRIND_MAKE_MEM_NOACCESS(((char *) chunk) + Slab_CHUNKHDRSZ +
slab->chunkSize,
slab->fullChunkSize -
(slab->chunkSize + Slab_CHUNKHDRSZ));
#endif
#ifdef RANDOMIZE_ALLOCATED_MEMORY
/* fill the allocated space with junk */
randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
#endif
/* Disallow access to the chunk header. */
VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
return MemoryChunkGetPointer(chunk);
}
pg_noinline
static void *
SlabAllocFromNewBlock(MemoryContext context, Size size, int flags)
{
SlabContext *slab = (SlabContext *) context;
SlabBlock *block;
MemoryChunk *chunk;
dlist_head *blocklist;
int blocklist_idx;
/* to save allocating a new one, first check the empty blocks list */
if (dclist_count(&slab->emptyblocks) > 0)
{
dlist_node *node = dclist_pop_head_node(&slab->emptyblocks);
block = dlist_container(SlabBlock, node, node);
/*
* SlabFree() should have left this block in a valid state with all
* chunks free. Ensure that's the case.
*/
Assert(block->nfree == slab->chunksPerBlock);
/* fetch the next chunk from this block */
chunk = SlabGetNextFreeChunk(slab, block);
}
else
{
block = (SlabBlock *) malloc(slab->blockSize);
if (unlikely(block == NULL))
return MemoryContextAllocationFailure(context, size, flags);
block->slab = slab;
context->mem_allocated += slab->blockSize;
/* use the first chunk in the new block */
chunk = SlabBlockGetChunk(slab, block, 0);
block->nfree = slab->chunksPerBlock - 1;
block->unused = SlabBlockGetChunk(slab, block, 1);
block->freehead = NULL;
block->nunused = slab->chunksPerBlock - 1;
}
/* find the blocklist element for storing blocks with 1 used chunk */
blocklist_idx = SlabBlocklistIndex(slab, block->nfree);
blocklist = &slab->blocklist[blocklist_idx];
/* this better be empty. We just added a block thinking it was */
Assert(dlist_is_empty(blocklist));
dlist_push_head(blocklist, &block->node);
slab->curBlocklistIndex = blocklist_idx;
return SlabAllocSetupNewChunk(context, block, chunk, size);
}
/*
* SlabAllocInvalidSize
* Handle raising an ERROR for an invalid size request. We don't do this
* in slab alloc as calling the elog functions would force the compiler
* to setup the stack frame in SlabAlloc. For performance reasons, we
* want to avoid that.
*/
pg_noinline
static void
pg_attribute_noreturn()
SlabAllocInvalidSize(MemoryContext context, Size size)
{
SlabContext *slab = (SlabContext *) context;
elog(ERROR, "unexpected alloc chunk size %zu (expected %u)", size,
slab->chunkSize);
}
/*
* SlabAlloc
* Returns a pointer to allocated memory of given size or NULL if
* request could not be completed; memory is added to the slab.
* Returns a pointer to a newly allocated memory chunk or raises an ERROR
* on allocation failure, or returns NULL when flags contains
* MCXT_ALLOC_NO_OOM. 'size' must be the same size as was specified
* during SlabContextCreate().
*
* This function should only contain the most common code paths. Everything
* else should be in pg_noinline helper functions, thus avoiding the overhead
* of creating a stack frame for the common cases. Allocating memory is often
* a bottleneck in many workloads, so avoiding stack frame setup is
* worthwhile. Helper functions should always directly return the newly
* allocated memory so that we can just return that address directly as a tail
* call.
*/
void *
SlabAlloc(MemoryContext context, Size size, int flags)
@ -513,66 +644,16 @@ SlabAlloc(MemoryContext context, Size size, int flags)
* MemoryContextCheckSize check.
*/
if (unlikely(size != slab->chunkSize))
elog(ERROR, "unexpected alloc chunk size %zu (expected %u)",
size, slab->chunkSize);
SlabAllocInvalidSize(context, size);
/*
* Handle the case when there are no partially filled blocks available.
* SlabFree() will have updated the curBlocklistIndex setting it to zero
* to indicate that it has freed the final block. Also later in
* SlabAlloc() we will set the curBlocklistIndex to zero if we end up
* filling the final block.
*/
if (unlikely(slab->curBlocklistIndex == 0))
{
dlist_head *blocklist;
int blocklist_idx;
/* to save allocating a new one, first check the empty blocks list */
if (dclist_count(&slab->emptyblocks) > 0)
{
dlist_node *node = dclist_pop_head_node(&slab->emptyblocks);
block = dlist_container(SlabBlock, node, node);
/*
* SlabFree() should have left this block in a valid state with
* all chunks free. Ensure that's the case.
*/
Assert(block->nfree == slab->chunksPerBlock);
/* fetch the next chunk from this block */
chunk = SlabGetNextFreeChunk(slab, block);
}
else
{
block = (SlabBlock *) malloc(slab->blockSize);
if (unlikely(block == NULL))
return MemoryContextAllocationFailure(context, size, flags);
block->slab = slab;
context->mem_allocated += slab->blockSize;
/* use the first chunk in the new block */
chunk = SlabBlockGetChunk(slab, block, 0);
block->nfree = slab->chunksPerBlock - 1;
block->unused = SlabBlockGetChunk(slab, block, 1);
block->freehead = NULL;
block->nunused = slab->chunksPerBlock - 1;
}
/* find the blocklist element for storing blocks with 1 used chunk */
blocklist_idx = SlabBlocklistIndex(slab, block->nfree);
blocklist = &slab->blocklist[blocklist_idx];
/* this better be empty. We just added a block thinking it was */
Assert(dlist_is_empty(blocklist));
dlist_push_head(blocklist, &block->node);
slab->curBlocklistIndex = blocklist_idx;
/*
* Handle the case when there are no partially filled blocks
* available. This happens either when the last allocation took the
* last chunk in the block, or when SlabFree() free'd the final block.
*/
return SlabAllocFromNewBlock(context, size, flags);
}
else
{
@ -609,38 +690,7 @@ SlabAlloc(MemoryContext context, Size size, int flags)
}
}
/*
* Check that the chunk pointer is actually somewhere on the block and is
* aligned as expected.
*/
Assert(chunk >= SlabBlockGetChunk(slab, block, 0));
Assert(chunk <= SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1));
Assert(SlabChunkMod(slab, block, chunk) == 0);
/* Prepare to initialize the chunk header. */
VALGRIND_MAKE_MEM_UNDEFINED(chunk, Slab_CHUNKHDRSZ);
MemoryChunkSetHdrMask(chunk, block, MAXALIGN(slab->chunkSize),
MCTX_SLAB_ID);
#ifdef MEMORY_CONTEXT_CHECKING
/* slab mark to catch clobber of "unused" space */
Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
set_sentinel(MemoryChunkGetPointer(chunk), size);
VALGRIND_MAKE_MEM_NOACCESS(((char *) chunk) +
Slab_CHUNKHDRSZ + slab->chunkSize,
slab->fullChunkSize -
(slab->chunkSize + Slab_CHUNKHDRSZ));
#endif
#ifdef RANDOMIZE_ALLOCATED_MEMORY
/* fill the allocated space with junk */
randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
#endif
/* Disallow access to the chunk header. */
VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
return MemoryChunkGetPointer(chunk);
return SlabAllocSetupNewChunk(context, block, chunk, size);
}
/*