memcached数据迁移问题及解决方案
相信不少朋友在迁移数据、扩容存储时需要将memcached中的数据导一份出来，然后在新的memcached中写入，操作步骤如下：第一步：导出所有key
方法一:memdump工具
memdump --servers="xxx.xxx.xxx.xxx:port" > keys.txt
方法二:
echo "stats cachedump slabid 0" | nc xxx.xxx.xxx.xxx port > keys.txt
第二步：获取所有key对应的value
第三步：将所有的key-value写入新的memcached
问题出来了，当我们在执行第一步时，发现导出的key的数量并不和memcached的number数一样多，如下图：
看了一下memdump命令帮助，发现文档里面描述了不能保证获取到所有的keys，另外查看memdump工具的源码，发现同样是使用stats cachedump命令实现的
于是就看了一下memcached的源码，发现有缓冲区大小限制，最大2M，汗
修改方案：
1、加大缓冲区，但该方法有局限性，当cache中的keys数太大时又面临缓冲区不够用的情况
2、修改stats cachedump命令，使其支持范围导出, stats cachedump slabid start limit
下面修改的代码，仅供参考：
Index: items.c
===================================================================
--- items.c (revision 793)
+++ items.c (working copy)
@@ -276,18 +276,23 @@
}
/*@null@*/
-char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes) {
+char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int start, const unsigned int limit, unsigned int *bytes) {
unsigned int memlimit = 2 * 1024 * 1024; /* 2MB max response size */
char *buffer;
unsigned int bufcurr;
item *it;
+ int i;
unsigned int len;
unsigned int shown = 0;
char temp;
if (slabs_clsid > LARGEST_ID) return NULL;
it = heads;
-
+ i = 0;
+ while (it != NULL && i < start) {
+ it = it->next;
+ i++;
+ }
buffer = malloc((size_t)memlimit);
if (buffer == 0) return NULL;
bufcurr = 0;
Index: items.h
===================================================================
--- items.h (revision 793)
+++ items.h (working copy)
@@ -12,7 +12,7 @@
intdo_item_replace(item *it, item *new_it);
/*@null@*/
-char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes);
+char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int start, const unsigned int limit, unsigned int *bytes);
char *do_item_stats(int *bytes);
/*@null@*/
Index: memcached.c
===================================================================
--- memcached.c (revision 793)
+++ memcached.c (working copy)
@@ -1115,7 +1115,7 @@
if (strcmp(subcommand, "cachedump") == 0) {
char *buf;
- unsigned int bytes, id, limit = 0;
+ unsigned int bytes, id, limit = 0, start = 0;
if(ntokens < 5) {
out_string(c, "CLIENT_ERROR bad command line");
@@ -1123,14 +1123,21 @@
}
id = strtoul(tokens.value, NULL, 10);
- limit = strtoul(tokens.value, NULL, 10);
+ if(ntokens == 5){
+ start = 0;
+ limit = strtoul(tokens.value, NULL, 10);
+ }
+ else if(ntokens == 6){
+ start = strtoul(tokens.value, NULL, 10);
+ limit = strtoul(tokens.value, NULL, 10);
+ }
if(errno == ERANGE) {
out_string(c, "CLIENT_ERROR bad command line format");
return;
}
- buf = item_cachedump(id, limit, &bytes);
+ buf = item_cachedump(id, start, limit, &bytes);
write_and_free(c, buf, bytes);
return;
}
Index: memcached.h
===================================================================
--- memcached.h (revision 793)
+++ memcached.h (working copy)
@@ -280,7 +280,7 @@
char *mt_defer_delete(item *it, time_t exptime);
int mt_is_listen_thread(void);
item *mt_item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes);
-char *mt_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes);
+char *mt_item_cachedump(const unsigned int slabs_clsid, const unsigned int start, const unsigned int limit, unsigned int *bytes);
voidmt_item_flush_expired(void);
item *mt_item_get_notedeleted(const char *key, const size_t nkey, bool *delete_locked);
int mt_item_link(item *it);
@@ -309,7 +309,7 @@
# define defer_delete(x,y) mt_defer_delete(x,y)
# define is_listen_thread() mt_is_listen_thread()
# define item_alloc(x,y,z,a,b) mt_item_alloc(x,y,z,a,b)
-# define item_cachedump(x,y,z) mt_item_cachedump(x,y,z)
+# define item_cachedump(x,y,z,a) mt_item_cachedump(x,y,z,a)
# define item_flush_expired() mt_item_flush_expired()
# define item_get_notedeleted(x,y,z) mt_item_get_notedeleted(x,y,z)
# define item_link(x) mt_item_link(x)
@@ -342,7 +342,7 @@
# define dispatch_event_add(t,c) event_add(&(c)->event, 0)
# define is_listen_thread() 1
# define item_alloc(x,y,z,a,b) do_item_alloc(x,y,z,a,b)
-# define item_cachedump(x,y,z) do_item_cachedump(x,y,z)
+# define item_cachedump(x,y,z,a) do_item_cachedump(x,y,z,a)
# define item_flush_expired() do_item_flush_expired()
# define item_get_notedeleted(x,y,z) do_item_get_notedeleted(x,y,z)
# define item_link(x) do_item_link(x)
Index: thread.c
===================================================================
--- thread.c (revision 793)
+++ thread.c (working copy)
@@ -528,11 +528,11 @@
/*
* Dumps part of the cache
*/
-char *mt_item_cachedump(unsigned int slabs_clsid, unsigned int limit, unsigned int *bytes) {
+char *mt_item_cachedump(unsigned int slabs_clsid, const unsigned int start, unsigned int limit, unsigned int *bytes) {
char *ret;
pthread_mutex_lock(&cache_lock);
- ret = do_item_cachedump(slabs_clsid, limit, bytes);
+ ret = do_item_cachedump(slabs_clsid, start, limit, bytes);
pthread_mutex_unlock(&cache_lock);
return ret;
}
页:
[1]