memcache slabs

11lxm

　　memcache 使用slab机制对内存进行分配和管理的。
　　
　　一 整体结构

　　
　　把内存分成多个层次，在每个层次中，可以有多个page, 所有层次的page大小基本是一直的（为什么不相同，后面介绍），每个page中又分成相同大小的item，里面有多少个item，取决于page大小和item所在的层次，其中相邻层次直接item大小具有如下关系
　　size(item: n+1 )  = size( item: n) * grow_factor   
　　其中的grow_factor 是可配置的参数。
　　在内存分配时，如果需要大小为k的空间，那么就从第n个层次中得到一个item ,其中n需要满足
　　size(item:n-1)  <   k  <=  size(item:n)
　　那么从n个层次中的这么多个item中取哪个？所有被取光了怎么办？使用完了释放了怎么办？ 这些都会在后面进行详细的介绍
　　
　　
　　二 结构成员
　　
　　typedef struct {unsigned int size;          /* 每个item大小, sizes of items */unsigned int perslab;       /* 每个page中包含多少个item , how many items per slab */void **slots;               /* 空闲的item指针, list of item ptrs */unsigned int sl_total;      /* 以分配空闲的item 个数, size of previous array */unsigned int sl_curr;       /* 当前空闲的item位置(也就是实际空闲item个数)，从后往前的， first free slot */void *end_page_ptr;         /* 指向最后一个页面中空闲的item开始位置, pointer to next free item at end of page, or 0 */unsigned int end_page_free; /* 最后一个页面，item个数, number of items remaining at end of last alloced page */unsigned int slabs;         /* 实际使用slab(page)个数 how many slabs were allocated for this class */void **slab_list;           /* 所有page的指针, array of slab pointers */unsigned int list_size;     /* 已经分配page指针个数，size of prev array */unsigned int killing;       /* index+1 of dying slab, or zero if none */size_t requested;           /* 所有被使用了的内存的大小, The number of requested bytes */} slabclass_t;
　　
　　三 函数介绍
　　
　　3.1 slabs_clsid()
　　/** Figures out which slab class (chunk size) is required to store an item of* a given size.** Given object size, return id to use when allocating/freeing memory for object* 0 means error: can't store such a large object* 根据大小，返回满足条件的item所在的层次 */unsigned int slabs_clsid(const size_t size) {int res = POWER_SMALLEST;if (size == 0)return 0;while (size > slabclass[res].size)if (res++ == power_largest)     /* won't fit in the biggest slab */return 0;return res;}
　　
　　3.2 slabs_init
　　/*** Determines the chunk sizes and initializes the slab class descriptors* accordingly.* slabs 初始化   1 如果需要预先分配的，那就预先分配内存 2* 对每个类别的slab，初始化该slab中每个item大小，item个数  3* 调用slabs_preallocate 预先分配slabs(给每个层次创建一个page) */void slabs_init(const size_t limit, const double factor, const bool prealloc) {int i = POWER_SMALLEST - 1;unsigned int size = sizeof(item) + settings.chunk_size;mem_limit = limit;if (prealloc) {//预先分配内存，并把开始地址和可使用空间大小记录/* Allocate everything in a big chunk with malloc */mem_base = malloc(mem_limit);if (mem_base != NULL) {mem_current = mem_base;mem_avail = mem_limit;} else {fprintf(stderr, "Warning: Failed to allocate requested memory in"" one large chunk./nWill allocate in smaller chunks/n");}}memset(slabclass, 0, sizeof(slabclass));while (++i < POWER_LARGEST && size <= settings.item_size_max / factor) {/* Make sure items are always n-byte aligned *///对齐if (size % CHUNK_ALIGN_BYTES)size += CHUNK_ALIGN_BYTES - (size % CHUNK_ALIGN_BYTES);//每个item大小以及item个数slabclass.size = size;slabclass.perslab = settings.item_size_max / slabclass.size;size *= factor;if (settings.verbose > 1) {fprintf(stderr, "slab class %3d: chunk size %9u perslab %7u/n",i, slabclass.size, slabclass.perslab);}}//最后一个slabpower_largest = i;slabclass[power_largest].size = settings.item_size_max;slabclass[power_largest].perslab = 1;if (settings.verbose > 1) {fprintf(stderr, "slab class %3d: chunk size %9u perslab %7u/n",i, slabclass.size, slabclass.perslab);}/* for the test suite:  faking of how much we've already malloc'd */{char *t_initial_malloc = getenv("T_MEMD_INITIAL_MALLOC");if (t_initial_malloc) {mem_malloced = (size_t)atol(t_initial_malloc);}}#ifndef DONT_PREALLOC_SLABS{char *pre_alloc = getenv("T_MEMD_SLABS_ALLOC");if (pre_alloc == NULL || atoi(pre_alloc) != 0) {slabs_preallocate(power_largest);}}#endif}
　　
　　3.3 slabs_preallocate
　　//对于每个层次的slab，预先分配一个pagestatic void slabs_preallocate (const unsigned int maxslabs) {int i;unsigned int prealloc = 0;/* pre-allocate a 1MB slab in every size class so people don't getconfused by non-intuitive "SERVER_ERROR out of memory"messages.  this is the most common question on the mailinglist.  if you really don't want this, you can rebuild withoutthese three lines.  */for (i = POWER_SMALLEST; i <= POWER_LARGEST; i++) {if (++prealloc > maxslabs)return;do_slabs_newslab(i);}}
　　
　　3.4  grow_slab_list
　　/确保page指针空间有空闲空间。如果可使用的空间满了，那么扩大成原来的两倍static int grow_slab_list (const unsigned int id) {slabclass_t *p = &slabclass[id];if (p->slabs == p->list_size) {size_t new_size =  (p->list_size != 0) ? p->list_size * 2 : 16;void *new_list = realloc(p->slab_list, new_size * sizeof(void *));if (new_list == 0) return 0;p->list_size = new_size;p->slab_list = new_list;}return 1;}
　　
　　
　　3.5 do_slabs_newslab
　　//分配一个类别id为id的page//1 分配空间 2 最后page赋值，以及该page空闲item个数  3 page数目++//，把page指针指向当前page static int do_slabs_newslab(const unsigned int id) {slabclass_t *p = &slabclass[id];int len = p->size * p->perslab;char *ptr;// 1有内存限制并且超过了限制  2 并且确认page指针空间失败 3 分配一个page的内存空间失败if ((mem_limit && mem_malloced + len > mem_limit && p->slabs > 0) ||(grow_slab_list(id) == 0) ||((ptr = memory_allocate((size_t)len)) == 0)) {MEMCACHED_SLABS_SLABCLASS_ALLOCATE_FAILED(id);return 0;}memset(ptr, 0, (size_t)len);p->end_page_ptr = ptr;p->end_page_free = p->perslab;p->slab_list[p->slabs++] = ptr;mem_malloced += len;MEMCACHED_SLABS_SLABCLASS_ALLOCATE(id);return 1;}
　　
　　
　　3.6  do_slabs_alloc
　　//根据id，返回对应类别的一个item空间//按照下面优先级进行分配： 1 首先从空闲的空间(回收的) 2 从最后的page中获取 3//新分配一个page  static void *do_slabs_alloc(const size_t size, unsigned int id) {slabclass_t *p;void *ret = NULL;if (id < POWER_SMALLEST || id > power_largest) {MEMCACHED_SLABS_ALLOCATE_FAILED(size, 0);return NULL;}p = &slabclass[id];assert(p->sl_curr == 0 || ((item *)p->slots[p->sl_curr - 1])->slabs_clsid == 0);#ifdef USE_SYSTEM_MALLOCif (mem_limit && mem_malloced + size > mem_limit) {MEMCACHED_SLABS_ALLOCATE_FAILED(size, id);return 0;}mem_malloced += size;ret = malloc(size);MEMCACHED_SLABS_ALLOCATE(size, id, 0, ret);return ret;#endif/* fail unless we have space at the end of a recently allocated page,we have something on our freelist, or we could allocate a new page */// 1 最后面的页面有空间 2 空闲的地方有空间 3 分配一个新的页面if (! (p->end_page_ptr != 0 || p->sl_curr != 0 ||do_slabs_newslab(id) != 0)) {/* We don't have more memory available */ret = NULL;} else if (p->sl_curr != 0) {/* return off our freelist *///从空闲(回收)地方分配ret = p->slots[--p->sl_curr];} else {/* if we recently allocated a whole page, return from that */assert(p->end_page_ptr != NULL);ret = p->end_page_ptr;if (--p->end_page_free != 0) {p->end_page_ptr = ((caddr_t)p->end_page_ptr) + p->size;} else {p->end_page_ptr = 0;}}if (ret) {p->requested += size;MEMCACHED_SLABS_ALLOCATE(size, id, p->size, ret);} else {MEMCACHED_SLABS_ALLOCATE_FAILED(size, id);}return ret;}
　　
　　3.7 do_slabs_free
　　//释放一个item空间//把释放的空间放到空闲的slot中，不过有可能出现空闲slot不够的情况，对slot进行扩大static void do_slabs_free(void *ptr, const size_t size, unsigned int id) {slabclass_t *p;assert(((item *)ptr)->slabs_clsid == 0);assert(id >= POWER_SMALLEST && id <= power_largest);if (id < POWER_SMALLEST || id > power_largest)return;MEMCACHED_SLABS_FREE(size, id, ptr);p = &slabclass[id];#ifdef USE_SYSTEM_MALLOCmem_malloced -= size;free(ptr);return;#endif//空闲slot空间不够，进行扩大if (p->sl_curr == p->sl_total) { /* need more space on the free list */int new_size = (p->sl_total != 0) ? p->sl_total * 2 : 16;  /* 16 is arbitrary */void **new_slots = realloc(p->slots, new_size * sizeof(void *));if (new_slots == 0)return;p->slots = new_slots;p->sl_total = new_size;}p->slots[p->sl_curr++] = ptr;p->requested -= size;return;}
　　
　　四 总结
　　
　　优点：  通过不同层次的item，以及空闲item维护，可以降低请求过程中内存的分配和释放，提高效率
　　
　　缺点： 1 每次使用的item可能大于时间需求空间量，导致内存浪费。
　　
　　
　　
　　五 参考
　　
　　http://blogold.iyunv.com/u1/56406/showart_2331017.html     memcache探索之slabs
　　
　　http://tech.idv2.com/2008/07/11/memcached-002/    理解memcache内存机制
　　
　　
　　
　　
             版权声明：本文为博主原创文章，未经博主允许不得转载。