zookeeper 的多线程和单线程库使用对比

鬼泣

　　
　　zookeeper提供了两个库，zookeeper_st和 zookeeper_mt。
　　前者是单线程库，仅仅提供了异步API和集成在应用程序实现循环中的回调函数，这个库是为了支持pthread库不支持或者不稳定的系统而存在的。使用过程中要自己通过zoo_interest和zoo_process实现事件处理及通知机制。
　　其他情况下应该使用后者多线程库。因为它同时支持同步和异步API。使用起来也方便很多，可以看我的例子。
　　The package includes two shared libraries: zookeeper_st and zookeeper_mt. The former only provides the asynchronous APIs and callbacks for integrating into the application's event loop. The only reason this library exists is to support the platforms were a pthread library is not available or is unstable (i.e. FreeBSD 4.x). In all other cases, application developers should link with zookeeper_mt, as it includes support for both Sync and Async API.
　　
　　OK，talk is cheap。
　　单线程库使用示例： 在zookeeper上先创建/xyz才能执行成功，因为st库仅支持异步，而异步create没有时序保证。 该程序可以监控/xyz节点及子节点的变化。



#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <zookeeper/zookeeper.h>
#include <zookeeper/zookeeper_log.h>
#include <errno.h>
#define TRUE 1
static const char* state2String(int state){
if (state == 0)
return "CLOSED_STATE";
if (state == ZOO_CONNECTING_STATE)
return "CONNECTING_STATE";
if (state == ZOO_ASSOCIATING_STATE)
return "ASSOCIATING_STATE";
if (state == ZOO_CONNECTED_STATE)
return "CONNECTED_STATE";
if (state == ZOO_EXPIRED_SESSION_STATE)
return "EXPIRED_SESSION_STATE";
if (state == ZOO_AUTH_FAILED_STATE)
return "AUTH_FAILED_STATE";
return "INVALID_STATE";
}
static const char* type2String(int state){
if (state == ZOO_CREATED_EVENT)
return "CREATED_EVENT";
if (state == ZOO_DELETED_EVENT)
return "DELETED_EVENT";
if (state == ZOO_CHANGED_EVENT)
return "CHANGED_EVENT";
if (state == ZOO_CHILD_EVENT)
return "CHILD_EVENT";
if (state == ZOO_SESSION_EVENT)
return "SESSION_EVENT";
if (state == ZOO_NOTWATCHING_EVENT)
return "NOTWATCHING_EVENT";
return "UNKNOWN_EVENT_TYPE";
}
void zktest_dump_stat(const struct Stat *stat) {
char tctimes[40];
char tmtimes[40];
time_t tctime;
time_t tmtime;
if (!stat) {
fprintf(stderr,"null\n");
return;
}
tctime = stat->ctime/1000;
tmtime = stat->mtime/1000;
ctime_r(&tmtime, tmtimes);
ctime_r(&tctime, tctimes);
fprintf(stderr, "\tctime = %s\tczxid=%llx\n"
"\tmtime=%s\tmzxid=%llx\n"
"\tversion=%x\taversion=%x\n"
"\tephemeralOwner = %llx\n",
tctimes, stat->czxid,
tmtimes, stat->mzxid,
(unsigned int)stat->version, (unsigned int)stat->aversion,
stat->ephemeralOwner);
}
void zktest_data_completion(int rc, const char *value, int value_len, const struct Stat *stat, const void *data) {
fprintf(stderr, "in data completion [%s]: rc = %d\n", value, rc);
zktest_dump_stat(stat);
}
/*process a list of string and stat*/
void zktest_strings_stat_completion(int rc, const struct String_vector *strings, const struct Stat *stat, const void *data) {
int i=0;
fprintf(stderr, "in strings state completion [%s]: rc = %d, string count %d\n", (char *)data, rc, strings ->count);
for (i=0; i<strings->count; i++) {
printf("%d: %s\n", i, strings->data);
}
deallocate_String_vector(strings);
zktest_dump_stat(stat);
}
void zktest_string_completion(int rc, const char *name, const void *data) {
fprintf(stderr, "in string completion [%s]: rc = %d\n", (char*)(data==0?"null":data), rc);
if (!rc) {
fprintf(stderr, "\tname = %s\n", name);
}
}
void zktest_stat_completion(int rc, const struct Stat *stat, const void *data) {
if (rc == ZNONODE)
{
printf("node not exists\n");
}
fprintf(stderr, "in state completion rc = %d data=%s stat:\n", rc, (char *)data);
zktest_dump_stat(stat);
}
void zktest_void_completion(int rc, const void *data) {
fprintf(stderr, "in void completion [%s]: rc = %d\n", (char*)(data==0?"null":data), rc);
}

void zktest_watcher_g(zhandle_t* zh, int type, int state, const char* path, void* watcherCtx) {
printf("Something happened.\n");
printf("type: %s\n", type2String(type));
printf("state: %s\n", state2String(state));
printf("path: %s\n", path);
printf("watcherCtx: %s\n", (char *)watcherCtx);
int ret;
const clientid_t *zk_clientid;
if (path == NULL || strlen(path) == 0)
{
path="/xyz";
}
if (type == ZOO_SESSION_EVENT)  {
if (state == ZOO_EXPIRED_SESSION_STATE) {
printf("[%s %d]zookeeper session expired\n", __FUNCTION__, __LINE__);
} else if (state == ZOO_CONNECTED_STATE) {
zk_clientid = zoo_client_id(zh);
printf("[%s %d] connected to zookeeper server with clientid=%lu\n", __FUNCTION__, __LINE__, zk_clientid ->client_id);
if ((ret=zoo_aexists(zh, path, TRUE, zktest_stat_completion, path)) != ZOK)
{
printf("[%s %d] zoo_aexists judge error, msg = %s\n", __FUNCTION__, __LINE__, zerror(ret));
}
if ((ret = zoo_aget(zh, (char *)path, TRUE, zktest_data_completion, "get param")) !=ZOK)
{
printf("[%s %d] fail to get znode %s, err=%d, msg= %s\n", __FUNCTION__, __LINE__, path, ret, zerror(ret));
}
if ((ret = zoo_aget_children2(zh, path, TRUE, zktest_strings_stat_completion, "get children param")) !=ZOK)
{
printf("[%s %d] fail to get znode %s, err=%d, msg= %s\n", __FUNCTION__, __LINE__, path, ret, zerror(ret));
}
}
} else if (type == ZOO_CREATED_EVENT) {
printf("create a child node\n");
if ((ret = zoo_aget_children2(zh, path, TRUE, zktest_strings_stat_completion, "get children param")) !=ZOK) {
printf("[%s %d] fail to get znode %s, err=%d, msg= %s\n", __FUNCTION__, __LINE__, path, ret, zerror(ret));
}
if ((ret = zoo_aget(zh, path, TRUE, zktest_data_completion, "get param")) !=ZOK) {
printf("[%s %d] fail to get znode %s, err=%d, msg= %s\n", __FUNCTION__, __LINE__, path, ret, zerror(ret));
}
} else if (type == ZOO_DELETED_EVENT) {
printf("path %s deleted\n", path);
if ((ret = zoo_aget(zh, path, TRUE, zktest_data_completion, "get param")) !=ZOK) {
printf("[%s %d] fail to get znode %s, err=%d, msg= %s\n", __FUNCTION__, __LINE__, path, ret, zerror(ret));
}
} else if (type == ZOO_CHANGED_EVENT) {
printf("path %s changed\n", path);
if ((ret = zoo_aget_children2(zh, path, TRUE, zktest_strings_stat_completion, "get children param")) !=ZOK) {
printf("[%s %d] fail to get znode %s, err=%d, msg= %s\n", __FUNCTION__, __LINE__, path, ret, zerror(ret));
}
if ((ret = zoo_aget(zh, path, TRUE, zktest_data_completion, "get param")) !=ZOK) {
printf("[%s %d] fail to get znode %s, err=%d, msg= %s\n", __FUNCTION__, __LINE__, path, ret, zerror(ret));
}
} else if (type == ZOO_CHILD_EVENT) {
printf("path %s child event\n");
if ((ret = zoo_aget(zh, path, TRUE, zktest_data_completion, "get param")) !=ZOK) {
printf("[%s %d] fail to get znode %s, err=%d, msg= %s\n", __FUNCTION__, __LINE__, path, ret, zerror(ret));
}
if ((ret = zoo_aget_children2(zh, path, TRUE, zktest_strings_stat_completion, "get children param")) !=ZOK) {
printf("[%s %d] fail to get znode %s, err=%d, msg= %s\n", __FUNCTION__, __LINE__, path, ret, zerror(ret));
}
}
else if (type == ZOO_NOTWATCHING_EVENT){
}
else {
printf("unkown zoo event type\n");
}
}

int main(int argc, const char *argv[])
{
static const char* host = "localhost:2181";
int timeout = 30000;
zhandle_t* zkhandle;
zoo_set_debug_level(ZOO_LOG_LEVEL_DEBUG);
zoo_deterministic_conn_order(1);
zkhandle = zookeeper_init(host, zktest_watcher_g, 30000, 0, "hello world", 0);
if (zkhandle == NULL) {
fprintf(stderr, "Error when connecting to zookeeper servers...\n");
exit(EXIT_FAILURE);
}
/* Wait for asynchronous zookeeper call done.*/
int fd, interest, events;
int rc;
fd_set rfds, wfds, efds;
FD_ZERO(&rfds);
FD_ZERO(&wfds);
FD_ZERO(&efds);
while(1){
struct timeval tv;
zookeeper_interest(zkhandle, &fd, &interest, &tv);
if (fd != -1) {
if(interest&ZOOKEEPER_READ)
{
FD_SET(fd, &rfds);
} else {
FD_CLR(fd, &rfds);
}
if(interest&ZOOKEEPER_WRITE)
{
FD_SET(fd, &wfds);
} else{
FD_CLR(fd, &wfds);
}
} else{
fd = 0;
}
if (select(fd+1, &rfds, &wfds, &efds, &tv) < 0)
{
printf("[%s %d]select failed, err=%d, msg=%s\n", __FUNCTION__, __LINE__, errno, strerror(errno));
}
events = 0;
if (FD_ISSET(fd, &rfds))
{
events |= ZOOKEEPER_READ;
}
if (FD_ISSET(fd, &wfds))
{
events |= ZOOKEEPER_WRITE;
}
zookeeper_process(zkhandle, events);
}
zookeeper_close(zkhandle);
}
　　
　　gcc -o zookeeper_s_test zookeeper_s_test.c -lzookeeper_st -I /usr/local/include/zookeeper/
　　
　　执行结果：



2014-10-21 22:05:41,856:49845:ZOO_INFO@zookeeper_init@786: Initiating client connection, host=localhost:2181 sessionTimeout=30000 watcher=0x401444 sessionId=0 sessionPasswd=<null> context=0x402213 flags=0
2014-10-21 22:05:41,859:49845:ZOO_INFO@check_events@1703: initiated connection to server [::1:2181]
2014-10-21 22:05:41,860:49845:ZOO_INFO@check_events@1750: session establishment complete on server [::1:2181], sessionId=0x14929440e3d0027, negotiated timeout=30000
2014-10-21 22:05:41,860:49845:ZOO_DEBUG@check_events@1756: Calling a watcher for a ZOO_SESSION_EVENT and the state=ZOO_CONNECTED_STATE
2014-10-21 22:05:41,860:49845:ZOO_DEBUG@process_completions@2107: Calling a watcher for node [], type = -1 event=ZOO_SESSION_EVENT
Something happened.
type: SESSION_EVENT
state: CONNECTED_STATE
path:
watcherCtx: hello world
[zktest_watcher_g 124] connected to zookeeper server with clientid=92650639611199527
2014-10-21 22:05:41,860:49845:ZOO_DEBUG@zoo_awexists@2835: Sending request xid=0x54473b25 for path [/xyz] to ::1:2181
2014-10-21 22:05:41,860:49845:ZOO_DEBUG@zoo_awget@2655: Sending request xid=0x54473b26 for path [/xyz] to ::1:2181
2014-10-21 22:05:41,860:49845:ZOO_DEBUG@zoo_awget_children2_@2916: Sending request xid=0x54473b27 for path [/xyz] to ::1:2181
2014-10-21 22:05:41,862:49845:ZOO_DEBUG@zookeeper_process@2264: Queueing asynchronous response
2014-10-21 22:05:41,862:49845:ZOO_DEBUG@deserialize_response@2005: Calling COMPLETION_STAT for xid=0x54473b25 failed=0 rc=0
in state completion rc = 0 data=/xyz stat:
ctime = Tue Oct 21 21:48:42 2014
czxid=400000052
mtime=Tue Oct 21 21:56:06 2014
mzxid=40000005a
version=2aversion=0
ephemeralOwner = 0
2014-10-21 22:05:41,862:49845:ZOO_DEBUG@zookeeper_process@2264: Queueing asynchronous response
2014-10-21 22:05:41,862:49845:ZOO_DEBUG@deserialize_response@1992: Calling COMPLETION_DATA for xid=0x54473b26 failed=0 rc=0
in data completion [123]: rc = 0
ctime = Tue Oct 21 21:48:42 2014
czxid=400000052
mtime=Tue Oct 21 21:56:06 2014
mzxid=40000005a
version=2aversion=0
ephemeralOwner = 0
2014-10-21 22:05:41,862:49845:ZOO_DEBUG@zookeeper_process@2264: Queueing asynchronous response
2014-10-21 22:05:41,862:49845:ZOO_DEBUG@deserialize_response@2029: Calling COMPLETION_STRINGLIST_STAT for xid=0x54473b27 failed=0 rc=0
in strings state completion [get children param]: rc = 0, string count 0
ctime = Tue Oct 21 21:48:42 2014
czxid=400000052
mtime=Tue Oct 21 21:56:06 2014
mzxid=40000005a
version=2aversion=0
ephemeralOwner = 0

　　
　　建议初学者使用时直接开debug，可以看清楚zookeeper的处理机制。
　　多线程库使用示例。该程序自己创建，读取，并删除一个节点，并能够触发watcher。


　　#include<stdio.h>
#include<string.h>
#include<zookeeper.h>
#include<zookeeper_log.h>
#include <net/if.h>
#include <netinet/in.h>
#include <sys/ioctl.h>
#include <arpa/inet.h>
　　static const char* state2String(int state){
if (state == 0)
return "CLOSED_STATE";
if (state == ZOO_CONNECTING_STATE)
return "CONNECTING_STATE";
if (state == ZOO_ASSOCIATING_STATE)
return "ASSOCIATING_STATE";
if (state == ZOO_CONNECTED_STATE)
return "CONNECTED_STATE";
if (state == ZOO_EXPIRED_SESSION_STATE)
return "EXPIRED_SESSION_STATE";
if (state == ZOO_AUTH_FAILED_STATE)
return "AUTH_FAILED_STATE";
　　return "INVALID_STATE";
}
　　static const char* type2String(int state){
if (state == ZOO_CREATED_EVENT)
return "CREATED_EVENT";
if (state == ZOO_DELETED_EVENT)
return "DELETED_EVENT";
if (state == ZOO_CHANGED_EVENT)
return "CHANGED_EVENT";
if (state == ZOO_CHILD_EVENT)
return "CHILD_EVENT";
if (state == ZOO_SESSION_EVENT)
return "SESSION_EVENT";
if (state == ZOO_NOTWATCHING_EVENT)
return "NOTWATCHING_EVENT";
　　return "UNKNOWN_EVENT_TYPE";
}
　　void zookeeper_watcher_g(zhandle_t* zh, int type, int state, const char* path, void* watcherCtx) {
printf("\nSomething happened.\n");
printf("type: %s\n", type2String(type));
printf("state: %s\n", state2String(state));
printf("path: %s\n", path);
printf("watcherCtx: %s\n", (char *)watcherCtx);
const clientid_t *zk_clientid;
int rc;
if (type == ZOO_CREATED_EVENT)
{
printf("[%s %d] znode %s created.\n", __FUNCTION__, __LINE__, path);
　　
} else if (type == ZOO_DELETED_EVENT){
printf("[%s %d] znode %s deleted.\n", __FUNCTION__, __LINE__, path);
　　
} else if (type == ZOO_CHANGED_EVENT){
printf("[%s %d] znode %s changed.\n", __FUNCTION__, __LINE__, path);

　　} else if(type == ZOO_CHILD_EVENT) {
printf("[%s %d] znode %s children changed.\n", __FUNCTION__, __LINE__, path);
　　
　　} else if(type == ZOO_SESSION_EVENT) {
if(state == ZOO_EXPIRED_SESSION_STATE){
printf("[%s %d] zookeeper session expired\n", __FUNCTION__, __LINE__);
} else if (state == ZOO_AUTH_FAILED_STATE) {
printf("[%s %d] zookeeper session auth failed\n", __FUNCTION__, __LINE__);
} else if (state == ZOO_CONNECTING_STATE)  {
printf("[%s %d] zookeeper session is connecting\n", __FUNCTION__, __LINE__);
} else if (state == ZOO_ASSOCIATING_STATE) {
printf("[%s %d] zookeeper session is associating state\n", __FUNCTION__, __LINE__);
} else if (state == ZOO_CONNECTED_STATE){
zk_clientid = zoo_client_id(zh);
printf("[%s %d] connected to zookeeper server with clientid=%lu\n", __FUNCTION__, __LINE__, zk_clientid ->client_id);
　　
} else if (state == ZOO_NOTWATCHING_EVENT) {
printf("[%s %d] zookeeper session remove watch\n", __FUNCTION__, __LINE__);
} else {
printf("unknown session event state = %s, path = %s, ctxt=%s\n", state2String(state), path, (char *)watcherCtx);
}
　　}
}
　　void create(zhandle_t *zkhandle,char *str) {
char path_buffer[64]="abc";
int bufferlen=sizeof(path_buffer);
printf("create node in synchronous mode-----------------------\n");
int flag = zoo_create(zkhandle, str,"syn-node", 9,
&ZOO_OPEN_ACL_UNSAFE,0,
path_buffer,bufferlen);
　　if (flag!=ZOK) {
printf("create syn-node failed\n");
exit(EXIT_FAILURE);
} else {
printf("created node is %s\n",path_buffer);
}
}
　　void exists(zhandle_t *zkhandle,char *path)
{
int flag = zoo_exists(zkhandle, path, 1, NULL);
if (flag) {
printf("%s node already exist\n", path);
} else {
printf("%s node not exist\n", path);
}
}
　　
　　void getACL(zhandle_t *zkhandle,char *str)
{
struct ACL_vector acl;
struct Stat stat;
int flag = zoo_get_acl(zkhandle,str,&acl,&stat);
if (flag==ZOK && acl.count > 0) {
printf("-----------------the ACL of %s:\n------------",str);
printf("%d\n",acl.count);
printf("%d\n",acl.data->perms);
printf("%s\n",acl.data->id.scheme);
printf("%s\n",acl.data->id.id);
}
}
void delete(zhandle_t *zkhandle,char *str)
{
int flag = zoo_delete(zkhandle,str,-1);
if (flag==ZOK){
printf("delete node %s success\n", str);
}
}
　　int main(int argc, const char *argv[])
{
if (argc != 2){
printf("usage: zookeeper_m_test  server_address\n");
exit(-1);
}
const char* host = argv[1];
int timeout = 30000;
char buffer[512];
int *bufferlen;
　　zoo_set_debug_level(ZOO_LOG_LEVEL_DEBUG);
zhandle_t* zkhandle = zookeeper_init(host, zookeeper_watcher_g, timeout, 0, NULL, 0);
if (zkhandle ==NULL) {
fprintf(stderr, "Error when connecting to zookeeper servers...\n");
exit(EXIT_FAILURE);
}
create(zkhandle, "/abc");
exists(zkhandle, "/abc");
getACL(zkhandle, "/abc");
delete(zkhandle, "/abc");
while(1);
zookeeper_close(zkhandle);
}
　　cc -o zookeeper_m_test zookeeper_m_test.o  -I/usr/local/include/zookeeper/ -L/usr/local/lib/ -L/usr/local/lib/x86_64-linux-gnu/ -lzookeeper_mt
　　执行：./zookeeper_m_test localhost:2181



2014-10-21 22:10:14,598:49903(0x7fc80aa16740):ZOO_INFO@zookeeper_init@786: Initiating client connection, host=localhost:2181 sessionTimeout=30000 watcher=0x400dfc sessionId=0 sessionPasswd=<null> context=(nil) flags=0
2014-10-21 22:10:14,598:49903(0x7fc80aa16740):ZOO_DEBUG@start_threads@221: starting threads...
2014-10-21 22:10:14,599:49903(0x7fc8094c2700):ZOO_DEBUG@do_io@367: started IO thread
2014-10-21 22:10:14,599:49903(0x7fc8094c2700):ZOO_INFO@check_events@1703: initiated connection to server [127.0.0.1:2181]
2014-10-21 22:10:14,599:49903(0x7fc808cc1700):ZOO_DEBUG@do_completion@459: started completion thread
create node in synchronous mode-----------------------
2014-10-21 22:10:14,599:49903(0x7fc80aa16740):ZOO_DEBUG@zoo_acreate@2756: Sending request xid=0x54473c37 for path [/abc] to 127.0.0.1:2181
2014-10-21 22:10:14,604:49903(0x7fc8094c2700):ZOO_INFO@check_events@1750: session establishment complete on server [127.0.0.1:2181], sessionId=0x14929440e3d0028, negotiated timeout=30000
2014-10-21 22:10:14,604:49903(0x7fc8094c2700):ZOO_DEBUG@check_events@1756: Calling a watcher for a ZOO_SESSION_EVENT and the state=ZOO_CONNECTED_STATE
2014-10-21 22:10:14,604:49903(0x7fc808cc1700):ZOO_DEBUG@process_completions@2107: Calling a watcher for node [], type = -1 event=ZOO_SESSION_EVENT
Something happened.
type: SESSION_EVENT
state: CONNECTED_STATE
path:
watcherCtx: (null)
[zookeeper_watcher_g 81] connected to zookeeper server with clientid=92650639611199528
2014-10-21 22:10:14,608:49903(0x7fc8094c2700):ZOO_DEBUG@process_sync_completion@1868: Processing sync_completion with type=6 xid=0x54473c37 rc=0
created node is /abc
2014-10-21 22:10:14,608:49903(0x7fc80aa16740):ZOO_DEBUG@zoo_awexists@2835: Sending request xid=0x54473c38 for path [/abc] to 127.0.0.1:2181
2014-10-21 22:10:14,609:49903(0x7fc8094c2700):ZOO_DEBUG@process_sync_completion@1868: Processing sync_completion with type=1 xid=0x54473c38 rc=0
/abc node not exist
2014-10-21 22:10:14,609:49903(0x7fc80aa16740):ZOO_DEBUG@zoo_aget_acl@2989: Sending request xid=0x54473c39 for path [/abc] to 127.0.0.1:2181
2014-10-21 22:10:14,611:49903(0x7fc8094c2700):ZOO_DEBUG@process_sync_completion@1868: Processing sync_completion with type=5 xid=0x54473c39 rc=0
-----------------the ACL of /abc:
------------1
31
world
anyone
2014-10-21 22:10:14,611:49903(0x7fc80aa16740):ZOO_DEBUG@zoo_adelete@2796: Sending request xid=0x54473c3a for path [/abc] to 127.0.0.1:2181
2014-10-21 22:10:14,613:49903(0x7fc8094c2700):ZOO_DEBUG@zookeeper_process@2193: Processing WATCHER_EVENT
2014-10-21 22:10:14,613:49903(0x7fc808cc1700):ZOO_DEBUG@process_completions@2107: Calling a watcher for node [/abc], type = -1 event=ZOO_DELETED_EVENT
Something happened.
type: DELETED_EVENT
state: CONNECTED_STATE
path: /abc
watcherCtx: (null)
[zookeeper_watcher_g 58] znode /abc deleted.
2014-10-21 22:10:14,613:49903(0x7fc8094c2700):ZOO_DEBUG@process_sync_completion@1868: Processing sync_completion with type=0 xid=0x54473c3a rc=0
delete node /abc success

　　很明显，多线程库使用起来更方便。
　　更多API使用详细可参考以下：
　　参考：
　　http://www.cnblogs.com/haippy/archive/2013/02/21/2920426.html
　　http://zookeeper.apache.org/doc/r3.4.6/zookeeperProgrammers.html#C+Binding