commit 4e682481051b96c25e9e997fc7cabe96298e945e Author: lericson Date: Mon Jul 27 16:10:43 2009 +0200 Initial import into git! diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8212673 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.pyc +*.pyo +build/* +dist/* +MANIFEST diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..84549d3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2008, Ludvig Ericson +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + - Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + - Neither the name of the author nor the names of the contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..4460877 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE MANIFEST _pylibmcmodule.c _pylibmcmodule.h pylibmc.py setup.py +include tests.py benchmark.py MANIFEST.in diff --git a/TODO b/TODO new file mode 100644 index 0000000..579e45a --- /dev/null +++ b/TODO @@ -0,0 +1,4 @@ +- double-check all INCREF and DECREFs +- make a get_stats function +- handle boolean values +- add servers after object creation (also related: set_servers) diff --git a/_pylibmcmodule.c b/_pylibmcmodule.c new file mode 100644 index 0000000..d23bf4a --- /dev/null +++ b/_pylibmcmodule.c @@ -0,0 +1,855 @@ +/** + * _pylibmc: hand-made libmemcached bindings for Python + * + * Copyright (c) 2008, Ludvig Ericson + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * - Neither the name of the author nor the names of the contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "_pylibmcmodule.h" + +/* {{{ _pylibmc.client implementation */ +/* {{{ Type methods */ +static PylibMC_Client *PylibMC_ClientType_new(PyTypeObject *type, + PyObject *args, PyObject *kwds) { + PylibMC_Client *self; + + self = (PylibMC_Client *)type->tp_alloc(type, 0); + if (self != NULL) { + self->mc = memcached_create(NULL); + } else { + args = kwds = NULL; + } + + return self; +} + +static void PylibMC_ClientType_dealloc(PylibMC_Client *self) { + memcached_free(self->mc); +} +/* }}} */ + +static int PylibMC_Client_init(PylibMC_Client *self, PyObject *args, + PyObject *kwds) { + PyObject *srv_list, *srv_list_it; + + if (!PyArg_ParseTuple(args, "O", &srv_list)) { + return -1; + } else { + kwds = NULL; + } + + if ((srv_list_it = PyObject_GetIter(srv_list)) != NULL) { + PyObject *c_srv; + + while ((c_srv = PyIter_Next(srv_list_it)) != NULL + && !PyErr_Occurred()) { + unsigned char stype; + char *hostname; + unsigned short int port; + + port = 0; + if (PyString_Check(c_srv)) { + memcached_server_st *list; + + list = memcached_servers_parse(PyString_AS_STRING(c_srv)); + if (list != NULL) { + memcached_return rc; + + rc = memcached_server_push(self->mc, list); + if (rc != MEMCACHED_SUCCESS) { + PylibMC_ErrFromMemcached(self, + "memcached_server_push", rc); + } + free(list); + } else { + PyErr_SetString(PylibMCExc_MemcachedError, + "memcached_servers_parse returned NULL"); + } + } else if (PyArg_ParseTuple(c_srv, "Bs|H", + &stype, &hostname, &port)) { + switch (stype) { + case PYLIBMC_SERVER_TCP: + memcached_server_add(self->mc, hostname, port); + break; + case PYLIBMC_SERVER_UDP: + memcached_server_add_udp(self->mc, hostname, port); + break; + case PYLIBMC_SERVER_UNIX: + if (port) { + PyErr_SetString(PyExc_ValueError, + "can't set port on unix sockets"); + } else { + memcached_server_add_unix_socket( + self->mc, hostname); + } + break; + default: + PyErr_Format(PyExc_ValueError, + "bad server type: %u", stype); + } + } + Py_DECREF(c_srv); + } + Py_DECREF(srv_list_it); + } + + return PyErr_Occurred() ? -1 : 0; +} + +static PyObject *_PylibMC_parse_memcached_value(char *value, size_t size, + uint32_t flags) { + PyObject *retval; + + retval = NULL; + switch (flags) { + case PYLIBMC_FLAG_PICKLE: + retval = _PylibMC_Unpickle(value, size); + break; + case PYLIBMC_FLAG_INTEGER: + case PYLIBMC_FLAG_LONG: + retval = PyInt_FromString(value, NULL, 10); + break; + case PYLIBMC_FLAG_NONE: + retval = PyString_FromStringAndSize(value, (Py_ssize_t)size); + break; + default: + PyErr_Format(PylibMCExc_MemcachedError, + "unknown memcached key flags %u", flags); + } + + return retval; +} + +static PyObject *PylibMC_Client_get(PylibMC_Client *self, PyObject *arg) { + char *mc_val; + size_t val_size; + uint32_t flags; + memcached_return error; + + if (!_PylibMC_CheckKey(arg)) { + return NULL; + } else if (!PySequence_Length(arg) ) { + /* Others do this, so... */ + Py_RETURN_NONE; + } + + mc_val = memcached_get(self->mc, + PyString_AS_STRING(arg), PyString_GET_SIZE(arg), + &val_size, &flags, &error); + if (mc_val != NULL) { + PyObject *r = _PylibMC_parse_memcached_value(mc_val, val_size, flags); + free(mc_val); + return r; + } + + if (error == MEMCACHED_NOTFOUND) { + /* Since python-memcache returns None when the key doesn't exist, + * so shall we. */ + Py_RETURN_NONE; + } + + return PylibMC_ErrFromMemcached(self, "memcached_get", error); +} + +/* {{{ Set commands (set, replace, add, prepend, append) */ +static PyObject *_PylibMC_RunSetCommand(PylibMC_Client *self, + _PylibMC_SetCommand f, char *fname, PyObject *args, + PyObject *kwds) { + char *key; + int key_len; + PyObject *val, *retval; + unsigned int time; + static char *kws[] = { "key", "val", "time", NULL }; + + retval = NULL; + time = 0; + if (PyArg_ParseTupleAndKeywords(args, kwds, "s#O|I", kws, + &key, &key_len, &val, &time)) { + PyObject *store_val = NULL; + uint32_t store_flags = 0; + + if (!_PylibMC_CheckKeyStringAndSize(key, key_len)) { + /* Let store_val be NULL, thus triggering an error. */ + } else if (PyString_Check(val)) { + store_val = val; + Py_INCREF(store_val); + } else if (PyInt_Check(val)) { + store_flags |= PYLIBMC_FLAG_INTEGER; + store_val = PyObject_Str(PyNumber_Int(val)); + } else if (PyLong_Check(val)) { + store_flags |= PYLIBMC_FLAG_LONG; + store_val = PyObject_Str(PyNumber_Long(val)); + } else { + Py_INCREF(val); + store_flags |= PYLIBMC_FLAG_PICKLE; + store_val = _PylibMC_Pickle(val); + Py_DECREF(val); + } + + if (store_val != NULL) { + memcached_return rc; + const char *raw_val; + size_t raw_val_len; + + raw_val = PyString_AS_STRING(store_val); + raw_val_len = PyString_GET_SIZE(store_val); + Py_DECREF(store_val); + + rc = f(self->mc, key, key_len, raw_val, raw_val_len, time, + store_flags); + switch (rc) { + case MEMCACHED_SUCCESS: + retval = Py_True; + break; + case MEMCACHED_FAILURE: + case MEMCACHED_NO_KEY_PROVIDED: + case MEMCACHED_BAD_KEY_PROVIDED: + case MEMCACHED_MEMORY_ALLOCATION_FAILURE: + case MEMCACHED_NOTSTORED: + retval = Py_False; + break; + default: + PylibMC_ErrFromMemcached(self, fname, rc); + } + } + } + + Py_XINCREF(retval); + return retval; +} + +/* These all just call _PylibMC_RunSetCommand with the appropriate arguments. + * In other words: bulk. */ +static PyObject *PylibMC_Client_set(PylibMC_Client *self, PyObject *args, + PyObject *kwds) { + return _PylibMC_RunSetCommand( + self, memcached_set, "memcached_set", args, kwds); +} + +static PyObject *PylibMC_Client_replace(PylibMC_Client *self, PyObject *args, + PyObject *kwds) { + return _PylibMC_RunSetCommand( + self, memcached_replace, "memcached_replace", args, kwds); +} + +static PyObject *PylibMC_Client_add(PylibMC_Client *self, PyObject *args, + PyObject *kwds) { + return _PylibMC_RunSetCommand( + self, memcached_add, "memcached_add", args, kwds); +} + +static PyObject *PylibMC_Client_prepend(PylibMC_Client *self, PyObject *args, + PyObject *kwds) { + return _PylibMC_RunSetCommand( + self, memcached_prepend, "memcached_prepend", args, kwds); +} + +static PyObject *PylibMC_Client_append(PylibMC_Client *self, PyObject *args, + PyObject *kwds) { + return _PylibMC_RunSetCommand( + self, memcached_append, "memcached_append", args, kwds); +} +/* }}} */ + +static PyObject *PylibMC_Client_delete(PylibMC_Client *self, PyObject *args) { + PyObject *retval; + char *key; + int key_len; + unsigned int time; + memcached_return rc; + + retval = NULL; + time = 0; + if (PyArg_ParseTuple(args, "s#|I", &key, &key_len, &time) + && _PylibMC_CheckKeyStringAndSize(key, key_len)) { + switch (rc = memcached_delete(self->mc, key, key_len, time)) { + case MEMCACHED_SUCCESS: + Py_RETURN_TRUE; + break; + case MEMCACHED_FAILURE: + case MEMCACHED_NOTFOUND: + case MEMCACHED_NO_KEY_PROVIDED: + case MEMCACHED_BAD_KEY_PROVIDED: + Py_RETURN_FALSE; + break; + default: + return PylibMC_ErrFromMemcached(self, "memcached_delete", rc); + } + } + + return NULL; +} + +/* {{{ Increment & decrement */ +static PyObject *_PylibMC_IncDec(PylibMC_Client *self, uint8_t dir, + PyObject *args) { + PyObject *retval; + char *key; + int key_len; + unsigned int delta; + uint64_t result; + + retval = NULL; + delta = 1; + if (PyArg_ParseTuple(args, "s#|I", &key, &key_len, &delta) + && _PylibMC_CheckKeyStringAndSize(key, key_len)) { + memcached_return (*incdec)(memcached_st *, const char *, size_t, + unsigned int, uint64_t *); + incdec = (dir == PYLIBMC_INC) ? memcached_increment + : memcached_decrement; + incdec(self->mc, key, key_len, delta, &result); + retval = PyLong_FromUnsignedLong((unsigned long)result); + } + + return retval; +} + +static PyObject *PylibMC_Client_incr(PylibMC_Client *self, PyObject *args) { + return _PylibMC_IncDec(self, PYLIBMC_INC, args); +} + +static PyObject *PylibMC_Client_decr(PylibMC_Client *self, PyObject *args) { + return _PylibMC_IncDec(self, PYLIBMC_DEC, args); +} +/* }}} */ + +static PyObject *PylibMC_Client_get_multi(PylibMC_Client *self, PyObject *args, + PyObject *kwds) { + PyObject *key_seq, **key_objs, *retval = NULL; + char **keys, *prefix = NULL; + unsigned int prefix_len = 0; + size_t *key_lens; + Py_ssize_t nkeys; + memcached_return rc; + + char curr_key[MEMCACHED_MAX_KEY]; + size_t curr_key_len, curr_val_len; + uint32_t curr_flags; + + static char *kws[] = { "keys", "key_prefix", NULL }; + + if (PyArg_ParseTupleAndKeywords(args, kwds, "O|s#", kws, + &key_seq, &prefix, &prefix_len)) { + PyObject *key_it, *ckey; + Py_ssize_t i; + + /* First clear any potential earlier mishap because we rely on it in + * our iteration over keys. */ + PyErr_Clear(); + /* Populate keys and key_lens. */ + nkeys = PySequence_Length(key_seq); + keys = malloc(sizeof(char *) * nkeys); + key_lens = malloc(sizeof(size_t) * nkeys); + key_objs = malloc(sizeof(PyObject *) * nkeys); + if (keys == NULL || key_lens == NULL || key_objs == NULL) { + free(keys); + free(key_lens); + free(key_objs); + return PyErr_NoMemory(); + } + /* Iterate through all keys and set lengths etc. */ + i = 0; + key_it = PyObject_GetIter(key_seq); + while (!PyErr_Occurred() + && i < nkeys + && (ckey = PyIter_Next(key_it)) != NULL) { + PyObject *rkey; + + if (!_PylibMC_CheckKey(ckey)) { + break; + } else { + key_lens[i] = PyString_GET_SIZE(ckey) + prefix_len; + if (prefix != NULL) { + rkey = PyString_FromFormat("%s%s", + prefix, PyString_AS_STRING(ckey)); + Py_DECREF(ckey); + } else { + rkey = ckey; + } + keys[i] = PyString_AS_STRING(rkey); + key_objs[i++] = rkey; + } + } + Py_DECREF(key_it); + + if (PyErr_Occurred()) { + free(key_lens); + free(keys); + nkeys = i; + for (i = 0; i < nkeys; i++) + Py_DECREF(key_objs[i]); + free(key_objs); + return NULL; + } + + /* TODO Change this interface or at least provide an alternative that + * returns some kind of iterable which fetches keys sequentially + * instead of doing it all at once. The current way is grossly + * inefficient for larger datasets, as a dict has to be allocated that + * is large enough to hold all the data at once. + */ + retval = PyDict_New(); + + if ((rc = memcached_mget(self->mc, keys, key_lens, nkeys)) + == MEMCACHED_SUCCESS) { + char *curr_val; + + while ((curr_val = memcached_fetch( + self->mc, curr_key, &curr_key_len, &curr_val_len, + &curr_flags, &rc)) != NULL + && !PyErr_Occurred()) { + if (curr_val == NULL && rc == MEMCACHED_END) { + break; + } else if (rc == MEMCACHED_BAD_KEY_PROVIDED + || rc == MEMCACHED_NO_KEY_PROVIDED) { + /* Do nothing at all. :-) */ + } else if (rc != MEMCACHED_SUCCESS) { + Py_DECREF(retval); + retval = PylibMC_ErrFromMemcached( + self, "memcached_fetch", rc); + } else { + PyObject *val; + + /* This is safe because libmemcached's max key length + * includes space for a NUL-byte. */ + curr_key[curr_key_len] = 0; + val = _PylibMC_parse_memcached_value( + curr_val, curr_val_len, curr_flags); + PyDict_SetItemString(retval, curr_key + prefix_len, val); + Py_DECREF(val); + } + free(curr_val); + } + /* Need to cleanup. */ + if (PyErr_Occurred()) { + /* Not checking rc because an exception already occured, and + * we wouldn't want to mask it. */ + memcached_quit(self->mc); + } + } else { + retval = PylibMC_ErrFromMemcached(self, "memcached_mget", rc); + } + + free(key_lens); + free(keys); + for (i = 0; i < nkeys; i++) + Py_DECREF(key_objs[i]); + free(key_objs); + } + + /* Not INCREFing because the only two outcomes are NULL and a new dict. + * We're the owner of that dict already, so. */ + return retval; +} + +/** + * Run func over every item in value, building arguments of: + * *(item + extra_args) + */ +static PyObject *_PylibMC_DoMulti(PyObject *values, PyObject *func, + PyObject *prefix, PyObject *extra_args) { + PyObject *retval = PyList_New(0); + PyObject *iter = NULL; + PyObject *item = NULL; + int is_mapping = PyMapping_Check(values); + + if (retval == NULL) + goto error; + + if ((iter = PyObject_GetIter(values)) == NULL) + goto error; + + while ((item = PyIter_Next(iter)) != NULL) { + PyObject *args_f = NULL; + PyObject *args = NULL; + PyObject *key = NULL; + PyObject *ro = NULL; + + /* Calculate key. */ + if (prefix == NULL || prefix == Py_None) { + /* We now have two owned references to item. */ + key = item; + Py_INCREF(key); + } else { + key = PySequence_Concat(prefix, item); + } + if (key == NULL || !_PylibMC_CheckKey(key)) + goto iter_error; + + /* Calculate args. */ + if (is_mapping) { + PyObject *value; + char *key_str = PyString_AS_STRING(item); + + if ((value = PyMapping_GetItemString(values, key_str)) == NULL) + goto iter_error; + + args = PyTuple_Pack(2, key, value); + } else { + args = PyTuple_Pack(1, key); + } + if (args == NULL) + goto iter_error; + + /* Calculate full argument tuple. */ + if (extra_args == NULL) { + Py_INCREF(args); + args_f = args; + } else { + if ((args_f = PySequence_Concat(args, extra_args)) == NULL) + goto iter_error; + } + + /* Call stuff. */ + ro = PyObject_CallObject(func, args_f); + /* This is actually safe even if True got deleted because we're + * only comparing addresses. */ + Py_XDECREF(ro); + if (ro == NULL) { + goto iter_error; + } else if (ro != Py_True) { + if (PyList_Append(retval, item) != 0) + goto iter_error; + } + Py_DECREF(args_f); + Py_DECREF(args); + Py_DECREF(key); + Py_DECREF(item); + continue; +iter_error: + Py_XDECREF(args_f); + Py_XDECREF(args); + Py_XDECREF(key); + Py_DECREF(item); + goto error; + } + Py_DECREF(iter); + + return retval; +error: + Py_XDECREF(retval); + Py_XDECREF(iter); + return NULL; +} + +static PyObject *PylibMC_Client_set_multi(PylibMC_Client *self, PyObject *args, + PyObject *kwds) { + PyObject *prefix = NULL; + PyObject *time = NULL; + PyObject *set = NULL; + PyObject *map; + PyObject *call_args; + PyObject *retval; + + static char *kws[] = { "mapping", "time", "key_prefix", NULL }; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O!S", kws, + &map, &PyInt_Type, &time, &prefix)) + return NULL; + + if ((set = PyObject_GetAttrString((PyObject *)self, "set")) == NULL) + return NULL; + + if (time == NULL) { + retval = _PylibMC_DoMulti(map, set, prefix, NULL); + } else { + if ((call_args = PyTuple_Pack(1, time)) == NULL) + goto error; + retval = _PylibMC_DoMulti(map, set, prefix, call_args); + Py_DECREF(call_args); + } + Py_DECREF(set); + + return retval; +error: + Py_XDECREF(set); + return NULL; +} + +static PyObject *PylibMC_Client_delete_multi(PylibMC_Client *self, + PyObject *args, PyObject *kwds) { + PyObject *prefix = NULL; + PyObject *time = NULL; + PyObject *delete; + PyObject *keys; + PyObject *call_args; + PyObject *retval; + + static char *kws[] = { "keys", "time", "key_prefix", NULL }; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O!S", kws, + &keys, &PyInt_Type, &time, &prefix)) + return NULL; + + if ((delete = PyObject_GetAttrString((PyObject *)self, "delete")) == NULL) + return NULL; + + if (time == NULL) { + retval = _PylibMC_DoMulti(keys, delete, prefix, NULL); + } else { + if ((call_args = PyTuple_Pack(1, time)) == NULL) + goto error; + retval = _PylibMC_DoMulti(keys, delete, prefix, call_args); + Py_DECREF(call_args); + } + Py_DECREF(delete); + + return retval; +error: + Py_XDECREF(delete); + return NULL; +} + +static PyObject *PylibMC_Client_get_behaviors(PylibMC_Client *self) { + PyObject *retval = PyDict_New(); + PylibMC_Behavior *b; + + for (b = PylibMC_behaviors; b->name != NULL; b++) { + uint64_t bval = memcached_behavior_get(self->mc, b->flag); + PyObject *x = PyInt_FromLong(bval); + + if (x == NULL || PyDict_SetItemString(retval, b->name, x)) { + goto error; + } + + Py_DECREF(x); + } + + return retval; +error: + Py_XDECREF(retval); + return NULL; +} + +static PyObject *PylibMC_Client_set_behaviors(PylibMC_Client *self, + PyObject *behaviors) { + PylibMC_Behavior *b; + + for (b = PylibMC_behaviors; b->name != NULL; b++) { + if (PyMapping_HasKeyString(behaviors, b->name)) { + PyObject *v = PyMapping_GetItemString(behaviors, b->name); + memcached_return r; + + if (!PyInt_Check(v)) { + PyErr_Format(PyExc_TypeError, "behavior %s must be int", + b->name); + goto error; + } + + r = memcached_behavior_set(self->mc, b->flag, PyInt_AS_LONG(v)); + Py_DECREF(v); + if (r != MEMCACHED_SUCCESS) { + PyErr_Format(PylibMCExc_MemcachedError, + "memcached_behavior_set returned %d", r); + goto error; + } + } + } + + Py_RETURN_NONE; +error: + return NULL; +} + +static PyObject *PylibMC_Client_flush_all(PylibMC_Client *self, + PyObject *args, PyObject *kwds) { + memcached_return rc; + time_t expire = 0; + PyObject *time = NULL; + + static char *kws[] = { "time", NULL }; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O!", kws, + &PyInt_Type, &time)) + return NULL; + + if (time != NULL) + expire = PyInt_AS_LONG(time); + + expire = (expire > 0) ? expire : 0; + + rc = memcached_flush(self->mc, expire); + if (rc != MEMCACHED_SUCCESS) + return PylibMC_ErrFromMemcached(self, "delete_multi", rc); + + Py_RETURN_TRUE; +} + +static PyObject *PylibMC_Client_disconnect_all(PylibMC_Client *self) { + memcached_quit(self->mc); + Py_RETURN_NONE; +} +/* }}} */ + +static PyObject *PylibMC_ErrFromMemcached(PylibMC_Client *self, const char *what, + memcached_return error) { + if (error == MEMCACHED_ERRNO) { + PyErr_Format(PylibMCExc_MemcachedError, + "system error %d from %s: %s", errno, what, strerror(errno)); + } else { + PyErr_Format(PylibMCExc_MemcachedError, "error %d from %s: %s", + error, what, memcached_strerror(self->mc, error)); + } + return NULL; +} + +/* {{{ Pickling */ +static PyObject *_PylibMC_GetPickles(const char *attname) { + PyObject *pickle, *pickle_attr; + + pickle_attr = NULL; + /* Import cPickle or pickle. */ + pickle = PyImport_ImportModule("cPickle"); + if (pickle == NULL) { + PyErr_Clear(); + pickle = PyImport_ImportModule("pickle"); + } + + /* Find attribute and return it. */ + if (pickle != NULL) { + pickle_attr = PyObject_GetAttrString(pickle, attname); + Py_DECREF(pickle); + } + return pickle_attr; +} + +static PyObject *_PylibMC_Unpickle(const char *buff, size_t size) { + PyObject *pickle_load; + PyObject *retval = NULL; + + retval = NULL; + pickle_load = _PylibMC_GetPickles("loads"); + if (pickle_load != NULL) { + retval = PyObject_CallFunction(pickle_load, "s#", buff, size); + Py_DECREF(pickle_load); + } + + return retval; +} + +static PyObject *_PylibMC_Pickle(PyObject *val) { + PyObject *pickle_dump; + PyObject *retval = NULL; + + retval = NULL; + pickle_dump = _PylibMC_GetPickles("dumps"); + if (pickle_dump != NULL) { + retval = PyObject_CallFunctionObjArgs(pickle_dump, val, + PyInt_FromLong(-1), NULL); + Py_DECREF(pickle_dump); + } + + return retval; +} +/* }}} */ + +static int _PylibMC_CheckKey(PyObject *key) { + if (key == NULL) { + PyErr_SetString(PyExc_ValueError, "key must be given"); + return 0; + } else if (!PyString_Check(key)) { + PyErr_SetString(PyExc_TypeError, "key must be an instance of str"); + return 0; + } + + return _PylibMC_CheckKeyStringAndSize( + PyString_AS_STRING(key), PyString_GET_SIZE(key)); +} + +static int _PylibMC_CheckKeyStringAndSize(char *key, int size) { + if (size > MEMCACHED_MAX_KEY) { + PyErr_Format(PyExc_ValueError, "key too long, max is %d", + MEMCACHED_MAX_KEY); + return 0; + } + /* TODO Check key contents here. */ + + return key != NULL; +} + +static PyMethodDef PylibMC_functions[] = { + {NULL, NULL, 0, NULL} +}; + +PyMODINIT_FUNC init_pylibmc(void) { + PyObject *module; + PylibMC_Behavior *b; + char name[128]; + + if (PyType_Ready(&PylibMC_ClientType) < 0) { + return; + } + + module = Py_InitModule3("_pylibmc", PylibMC_functions, + "Hand-made wrapper for libmemcached.\n\ +\n\ +You ought to look at python-memcached's documentation for now, seeing\n\ +as this module is more or less a drop-in replacement, the difference\n\ +being in how you connect. Therefore that's documented here::\n\ +\n\ + c = _pylibmc.client([(_pylibmc.server_type_tcp, 'localhost', 11211)])\n\ +\n\ +As you see, a list of three-tuples of (type, host, port) is used. If \n\ +type is `server_type_unix`, no port should be given. A simpler form \n\ +can be used as well::\n\ +\n\ + c = _pylibmc.client('localhost')\n\ +\n\ +See libmemcached's memcached_servers_parse for more info on that. I'm told \n\ +you can use UNIX domain sockets by specifying paths, and multiple servers \n\ +by using comma-separation. Good luck with that.\n\ +\n\ +Oh, and: plankton.\n"); + if (module == NULL) { + return; + } + + PylibMCExc_MemcachedError = PyErr_NewException( + "_pylibmc.MemcachedError", NULL, NULL); + + Py_INCREF(&PylibMC_ClientType); + PyModule_AddObject(module, "client", (PyObject *)&PylibMC_ClientType); + + PyModule_AddIntConstant(module, "server_type_tcp", PYLIBMC_SERVER_TCP); + PyModule_AddIntConstant(module, "server_type_udp", PYLIBMC_SERVER_UDP); + PyModule_AddIntConstant(module, "server_type_unix", PYLIBMC_SERVER_UNIX); + + /* Add hasher and distribution constants. */ + for (b = PylibMC_hashers; b->name != NULL; b++) { + sprintf(name, "hash_%s", b->name); + PyModule_AddIntConstant(module, name, b->flag); + } + for (b = PylibMC_distributions; b->name != NULL; b++) { + sprintf(name, "distribution_%s", b->name); + PyModule_AddIntConstant(module, name, b->flag); + } + + PyModule_AddStringConstant(module, + "libmemcached_version", LIBMEMCACHED_VERSION_STRING); +} diff --git a/_pylibmcmodule.h b/_pylibmcmodule.h new file mode 100644 index 0000000..bdd597b --- /dev/null +++ b/_pylibmcmodule.h @@ -0,0 +1,242 @@ +/** + * _pylibmc: hand-made libmemcached bindings for Python + * + * Copyright (c) 2008, Ludvig Ericson + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * - Neither the name of the author nor the names of the contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __PYLIBMC_H__ +#define __PYLIBMC_H__ + +#include +#include + +/* Server types. */ +#define PYLIBMC_SERVER_TCP (1 << 0) +#define PYLIBMC_SERVER_UDP (1 << 1) +#define PYLIBMC_SERVER_UNIX (1 << 2) + +/* Key flags from python-memcache. */ +#define PYLIBMC_FLAG_NONE 0 +#define PYLIBMC_FLAG_PICKLE (1 << 0) +#define PYLIBMC_FLAG_INTEGER (1 << 1) +#define PYLIBMC_FLAG_LONG (1 << 2) + +#define PYLIBMC_INC (1 << 0) +#define PYLIBMC_DEC (1 << 1) + +typedef memcached_return (*_PylibMC_SetCommand)(memcached_st *, const char *, + size_t, const char *, size_t, time_t, uint32_t); + +/* {{{ Exceptions */ +static PyObject *PylibMCExc_MemcachedError; +/* }}} */ + +/* {{{ Behavior statics */ +typedef struct { + int flag; + char *name; +} PylibMC_Behavior; + +static PylibMC_Behavior PylibMC_behaviors[] = { + { MEMCACHED_BEHAVIOR_NO_BLOCK, "no block" }, + { MEMCACHED_BEHAVIOR_TCP_NODELAY, "tcp_nodelay" }, + { MEMCACHED_BEHAVIOR_HASH, "hash" }, + { MEMCACHED_BEHAVIOR_KETAMA, "ketama" }, + { MEMCACHED_BEHAVIOR_DISTRIBUTION, "distribution" }, + { MEMCACHED_BEHAVIOR_SOCKET_SEND_SIZE, "socket send size" }, + { MEMCACHED_BEHAVIOR_SOCKET_RECV_SIZE, "socket recv size" }, + { MEMCACHED_BEHAVIOR_CACHE_LOOKUPS, "cache_lookups" }, + { MEMCACHED_BEHAVIOR_SUPPORT_CAS, "support_cas" }, + { MEMCACHED_BEHAVIOR_POLL_TIMEOUT, "poll_timeout" }, + { MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, "buffer_requests" }, + { MEMCACHED_BEHAVIOR_SORT_HOSTS, "sort_hosts" }, + { MEMCACHED_BEHAVIOR_VERIFY_KEY, "verify_key" }, + { MEMCACHED_BEHAVIOR_CONNECT_TIMEOUT, "connect_timeout" }, + { MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, "retry_timeout" }, + { MEMCACHED_BEHAVIOR_KETAMA_WEIGHTED, "ketama_weighted" }, + { MEMCACHED_BEHAVIOR_KETAMA_HASH, "ketama_hash" }, + { MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, "binary_protocol" }, + { MEMCACHED_BEHAVIOR_SND_TIMEOUT, "snd_timeout" }, + { MEMCACHED_BEHAVIOR_RCV_TIMEOUT, "rcv_timeout" }, + { MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT, "server_failure_limit" }, + { 0, NULL } +}; + +static PylibMC_Behavior PylibMC_hashers[] = { + { MEMCACHED_HASH_DEFAULT, "default" }, + { MEMCACHED_HASH_MD5, "md5" }, + { MEMCACHED_HASH_CRC, "crc" }, + { MEMCACHED_HASH_FNV1_64, "fnv1_64" }, + { MEMCACHED_HASH_FNV1A_64, "fnv1a_64" }, + { MEMCACHED_HASH_FNV1_32, "fnv1_32" }, + { MEMCACHED_HASH_FNV1A_32, "fnv1a_32" }, + { MEMCACHED_HASH_HSIEH, "hsieh" }, + { MEMCACHED_HASH_MURMUR, "murmur" }, + { 0, NULL } +}; + +static PylibMC_Behavior PylibMC_distributions[] = { + { MEMCACHED_DISTRIBUTION_MODULA, "modula" }, + { MEMCACHED_DISTRIBUTION_CONSISTENT, "consistent" }, + { MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA, "consistent_ketama" }, + { 0, NULL } +}; +/* }}} */ + +/* {{{ _pylibmc.client */ +typedef struct { + PyObject_HEAD + memcached_st *mc; +} PylibMC_Client; + +/* {{{ Prototypes */ +static PylibMC_Client *PylibMC_ClientType_new(PyTypeObject *, PyObject *, + PyObject *); +static void PylibMC_ClientType_dealloc(PylibMC_Client *); +static int PylibMC_Client_init(PylibMC_Client *, PyObject *, PyObject *); +static PyObject *PylibMC_Client_get(PylibMC_Client *, PyObject *arg); +static PyObject *PylibMC_Client_set(PylibMC_Client *, PyObject *, PyObject *); +static PyObject *PylibMC_Client_replace(PylibMC_Client *, PyObject *, PyObject *); +static PyObject *PylibMC_Client_add(PylibMC_Client *, PyObject *, PyObject *); +static PyObject *PylibMC_Client_prepend(PylibMC_Client *, PyObject *, PyObject *); +static PyObject *PylibMC_Client_append(PylibMC_Client *, PyObject *, PyObject *); +static PyObject *PylibMC_Client_delete(PylibMC_Client *, PyObject *); +static PyObject *PylibMC_Client_incr(PylibMC_Client *, PyObject *); +static PyObject *PylibMC_Client_decr(PylibMC_Client *, PyObject *); +static PyObject *PylibMC_Client_get_multi(PylibMC_Client *, PyObject *, PyObject *); +static PyObject *PylibMC_Client_set_multi(PylibMC_Client *, PyObject *, PyObject *); +static PyObject *PylibMC_Client_delete_multi(PylibMC_Client *, PyObject *, PyObject *); +static PyObject *PylibMC_Client_get_behaviors(PylibMC_Client *); +static PyObject *PylibMC_Client_set_behaviors(PylibMC_Client *, PyObject *); +static PyObject *PylibMC_Client_flush_all(PylibMC_Client *, PyObject *, PyObject *); +static PyObject *PylibMC_Client_disconnect_all(PylibMC_Client *); +static PyObject *PylibMC_ErrFromMemcached(PylibMC_Client *, const char *, + memcached_return); +static PyObject *_PylibMC_Unpickle(const char *, size_t); +static PyObject *_PylibMC_Pickle(PyObject *); +static int _PylibMC_CheckKey(PyObject *); +static int _PylibMC_CheckKeyStringAndSize(char *, int); +/* }}} */ + +/* {{{ Type's method table */ +static PyMethodDef PylibMC_ClientType_methods[] = { + {"get", (PyCFunction)PylibMC_Client_get, METH_O, + "Retrieve a key from a memcached."}, + {"set", (PyCFunction)PylibMC_Client_set, METH_VARARGS|METH_KEYWORDS, + "Set a key unconditionally."}, + {"replace", (PyCFunction)PylibMC_Client_replace, METH_VARARGS|METH_KEYWORDS, + "Set a key only if it exists."}, + {"add", (PyCFunction)PylibMC_Client_add, METH_VARARGS|METH_KEYWORDS, + "Set a key only if doesn't exist."}, + {"prepend", (PyCFunction)PylibMC_Client_prepend, METH_VARARGS|METH_KEYWORDS, + "Prepend data to a key."}, + {"append", (PyCFunction)PylibMC_Client_append, METH_VARARGS|METH_KEYWORDS, + "Append data to a key."}, + {"delete", (PyCFunction)PylibMC_Client_delete, METH_VARARGS, + "Delete a key."}, + {"incr", (PyCFunction)PylibMC_Client_incr, METH_VARARGS, + "Increment a key by a delta."}, + {"decr", (PyCFunction)PylibMC_Client_decr, METH_VARARGS, + "Decrement a key by a delta."}, + {"get_multi", (PyCFunction)PylibMC_Client_get_multi, + METH_VARARGS|METH_KEYWORDS, "Get multiple keys at once."}, + {"set_multi", (PyCFunction)PylibMC_Client_set_multi, + METH_VARARGS|METH_KEYWORDS, "Set multiple keys at once."}, + {"delete_multi", (PyCFunction)PylibMC_Client_delete_multi, + METH_VARARGS|METH_KEYWORDS, "Delete multiple keys at once."}, + {"get_behaviors", (PyCFunction)PylibMC_Client_get_behaviors, METH_NOARGS, + "Get behaviors dict."}, + {"set_behaviors", (PyCFunction)PylibMC_Client_set_behaviors, METH_O, + "Set behaviors dict."}, + {"flush_all", (PyCFunction)PylibMC_Client_flush_all, + METH_VARARGS|METH_KEYWORDS, "Flush all data on all servers."}, + {"disconnect_all", (PyCFunction)PylibMC_Client_disconnect_all, METH_NOARGS, + "Disconnect from all servers and reset own state."}, + {NULL, NULL, 0, NULL} +}; +/* }}} */ + +/* {{{ Type def */ +static PyTypeObject PylibMC_ClientType = { + PyObject_HEAD_INIT(NULL) + 0, + "client", + sizeof(PylibMC_Client), + 0, + (destructor)PylibMC_ClientType_dealloc, + + 0, + 0, + 0, + 0, + 0, + + 0, + 0, + 0, + + 0, + 0, + 0, + 0, + 0, + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + "memcached client type", + 0, + 0, + 0, + 0, + 0, + 0, + PylibMC_ClientType_methods, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + (initproc)PylibMC_Client_init, + 0, + (newfunc)PylibMC_ClientType_new, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 +}; + +/* }}} */ + +#endif /* def __PYLIBMC_H__ */ diff --git a/cmemcache_tests.py b/cmemcache_tests.py new file mode 100644 index 0000000..ee12d8e --- /dev/null +++ b/cmemcache_tests.py @@ -0,0 +1,67 @@ +import pylibmc +import unittest +import pickle + +class TestCmemcached(unittest.TestCase): + + def setUp(self): + self.mc = pylibmc.Client(["127.0.0.1:11211"]) + #self.mc = pylibmc.Client(["127.0.0.1:11211","127.0.0.1:11212", "127.0.0.1:11213", "127.0.0.1:11214", "127.0.0.1:11215"]) + + def testSetAndGet(self): + self.mc.set("num12345", 12345) + self.assertEqual(self.mc.get("num12345"), 12345) + self.mc.set("str12345", "12345") + self.assertEqual(self.mc.get("str12345"), "12345") + + def testDelete(self): + self.mc.set("str12345", "12345") + #delete return True on success, otherwise False + ret = self.mc.delete("str12345") + self.assertEqual(self.mc.get("str12345"), None) + self.assertEqual(ret, True) + + ret = self.mc.delete("hello world") + self.assertEqual(ret, False) + + def testGetMulti(self): + self.mc.set("a", "valueA") + self.mc.set("b", "valueB") + self.mc.set("c", "valueC") + result = self.mc.get_multi(["a", "b", "c", "", "hello world"]) + self.assertEqual(result, {'a':'valueA', 'b':'valueB', 'c':'valueC'}) + + def testBigGetMulti(self): + count = 10 + keys = ['key%d' % i for i in xrange(count)] + pairs = zip(keys, ['value%d' % i for i in xrange(count)]) + for key, value in pairs: + self.mc.set(key, value) + result = self.mc.get_multi(keys) + assert result == dict(pairs) + + def testFunnyDelete(self): + result= self.mc.delete("") + self.assertEqual(result, False) + + def testAppend(self): + self.mc.delete("a") + self.mc.set("a", "I ") + ret = self.mc.append("a", "Do") + print ret + result = self.mc.get("a") + print result + self.assertEqual(result, "I Do") + + def testPrepend(self): + self.mc.delete("a") + self.mc.set("a", "Do") + ret = self.mc.prepend("a", "I ") + print ret + result = self.mc.get("a") + print result + self.assertEqual(result, "I Do") + +if __name__ == '__main__': + unittest.main() + diff --git a/pylibmc.py b/pylibmc.py new file mode 100644 index 0000000..6bb3d21 --- /dev/null +++ b/pylibmc.py @@ -0,0 +1,86 @@ +"""`python-memcached`-compatible wrapper around `_pylibmc`. + +>>> mc = Client(["127.0.0.1"]) +>>> b = mc.behaviors +>>> b.keys() +['hash', 'connect timeout', 'cache lookups', 'buffer requests', 'verify key', 'support cas', 'poll timeout', 'no block', 'tcp nodelay', 'distribution', 'sort hosts'] +>>> b["hash"] +'default' +>>> b["hash"] = 'fnv1a_32' +>>> mc.behaviors["hash"] +'fnv1a_32' +""" + +import _pylibmc + +__all__ = ["hashers", "distributions", "Client"] + +hashers = {} +distributions = {} +# Not the prettiest way of doing things, but works well. +for name in dir(_pylibmc): + if name.startswith("hash_"): + hashers[name[5:]] = getattr(_pylibmc, name) + elif name.startswith("distribution_"): + distributions[name[13:].replace("_", " ")] = getattr(_pylibmc, name) + +hashers_rvs = dict((v, k) for (k, v) in hashers.iteritems()) +distributions_rvs = dict((v, k) for (k, v) in distributions.iteritems()) + +class BehaviorDict(dict): + def __init__(self, client, *args, **kwds): + super(BehaviorDict, self).__init__(*args, **kwds) + self.client = client + + def __setitem__(self, name, value): + super(BehaviorDict, self).__setitem__(name, value) + self.client.behaviors = self + + def update(self, *args, **kwds): + super(BehaviorDict, self).update(*args, **kwds) + self.client.behaviors = self + +class Client(_pylibmc.client): + def __init__(self, servers, *args, **kwds): + addr_tups = [] + for server in servers: + addr = server + port = 11211 + if server.startswith("udp:"): + stype = _pylibmc.server_type_udp + addr = addr[4:] + if ":" in server: + (addr, port) = addr.split(":", 1) + port = int(port) + elif ":" in server: + stype = _pylibmc.server_type_tcp + (addr, port) = server.split(":", 1) + port = int(port) + elif "/" in server: + stype = _pylibmc.server_type_unix + port = 0 + else: + stype = _pylibmc.server_type_tcp + addr_tups.append((stype, addr, port)) + super(Client, self).__init__(addr_tups) + + def get_behaviors(self): + behaviors = super(Client, self).get_behaviors() + behaviors["hash"] = hashers_rvs[behaviors["hash"]] + behaviors["distribution"] = distributions_rvs[behaviors["distribution"]] + return BehaviorDict(self, behaviors) + + def set_behaviors(self, behaviors): + # Morph some. + behaviors = behaviors.copy() + if behaviors.get("hash", None) in hashers: + behaviors["hash"] = hashers[behaviors["hash"]] + if behaviors.get("distribution") in distributions: + behaviors["distribution"] = distributions[behaviors["distribution"]] + return super(Client, self).set_behaviors(behaviors) + + behaviors = property(get_behaviors, set_behaviors) + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f04b1ca --- /dev/null +++ b/setup.py @@ -0,0 +1,83 @@ +"""`pylibmc` is a Python wrapper around the accompanying C Python extension +`_pylibmc`, which is a wrapper around `libmemcached` from TangentOrg. + +You have to install `libmemcached` separately, and have your compiler and +linker find the include files and libraries. + +With `libmemcached` installed and this package set up, the following basic +usage example should work:: + + >>> import pylibmc + >>> mc = pylibmc.Client(["127.0.0.1:11211"]) + >>> mc.set("foo", "Hello world!") + True + >>> mc.get("foo") + 'Hello world!' + +The API is pretty much `python-memcached`. Some parts of `libmemcached` aren't +exposed yet. I think. + +Behaviors +========= + +`libmemcached` has ways of telling it how to behave. You'll have to refer to +its documentation on what the different behaviors do. + +To change behaviors, quite simply:: + + >>> mc.behaviors["hash"] = "fnv1a_32" + +Comparison to other libraries +============================= + +Why use `pylibmc`? Because it's fast. + +http://lericson.blogg.se/code/2008/november/pylibmc-051.html + +Change Log +========== + +New in version 0.6 +------------------ + +Added compatibility with `libmemcached` 0.26, WRT error return codes. + +Added `flush_all` and `disconnect_all` methods. + +Now using the latest pickling protocol. + +New in version 0.5 +------------------ + +Fixed lots of memory leaks, and added support for `libmemcached` 0.23. + +Also made the code tighter in terms of compiler pendatics. + +New in version 0.4 +------------------ + +Renamed the C module to `_pylibmc`, and added lots of `libmemcached` constants +to it, as well as implemented behaviors. +""" + +import os +from distutils.core import setup, Extension + +incdirs = [] +libdirs = [] + +if "LIBMEMCACHED_DIR" in os.environ: + libdir = os.path.normpath(os.environ["LIBMEMCACHED_DIR"]) + incdirs.append(os.path.join(libdir, "include")) + libdirs.append(os.path.join(libdir, "lib")) + +pylibmc_ext = Extension("_pylibmc", ["_pylibmcmodule.c"], + libraries=["memcached"], + include_dirs=incdirs, library_dirs=libdirs) + +setup(name="pylibmc", version="0.6.1", + url="http://lericson.blogg.se/code/category/pylibmc.html", + author="Ludvig Ericson", author_email="ludvig.ericson@gmail.com", + license="3-clause BSD ", + description="libmemcached wrapper", long_description=__doc__, + ext_modules=[pylibmc_ext], py_modules=["pylibmc"]) diff --git a/tests.py b/tests.py new file mode 100644 index 0000000..e32bcd5 --- /dev/null +++ b/tests.py @@ -0,0 +1,137 @@ +"""Tests. They want YOU!! + +Basic functionality. +>>> c = _pylibmc.client([test_server]) +>>> c.set("test_key", 123) +True +>>> c.get("test_key") +123 +>>> c.get("test_key_2") +>>> c.delete("test_key") +True +>>> c.get("test_key") +>>> + +Now this section is because most other implementations ignore zero-keys. +>>> c.get("") +>>> c.set("", "hi") +False +>>> c.delete("") +False +>>> + +Multi functionality. +>>> c.set_multi({"a": 1, "b": 2, "c": 3}) +[] +>>> c.get_multi("abc").keys() == ["a", "c", "b"] +True +>>> c.delete_multi("abc") +[] +>>> c.get_multi("abc").keys() == [] +True +>>> c.set_multi(dict(zip("abc", "def")), key_prefix="test_") +[] +>>> list(sorted(c.get_multi("abc", key_prefix="test_").iteritems())) +[('a', 'd'), ('b', 'e'), ('c', 'f')] +>>> c.get("test_a") +'d' +>>> c.delete_multi("abc", key_prefix="test_") +[] +>>> bool(c.get_multi("abc", key_prefix="test_")) +False + +Zero-key-test-time! +>>> c.get_multi([""]) +{} +>>> c.delete_multi([""]) +[''] +>>> c.set_multi({"": "hi"}) +[''] + +Timed stuff. The reason we, at UNIX times, set it two seconds in the future and +then sleep for >3 is that memcached might round the time up and down and left +and yeah, so you know... +>>> from time import sleep, time +>>> c.set("hi", "steven", 1) +True +>>> c.get("hi") +'steven' +>>> sleep(1.1) +>>> c.get("hi") +>>> c.set("hi", "loretta", int(time()) + 2) +True +>>> c.get("hi") +'loretta' +>>> sleep(3.1) +>>> c.get("hi") +>>> + +Now for keys with funny types. +>>> c.set(1, "hi") +Traceback (most recent call last): + ... +TypeError: argument 1 must be string or read-only buffer, not int +>>> c.get(1) +Traceback (most recent call last): + ... +TypeError: key must be an instance of str +>>> c.set_multi({1: True}) +Traceback (most recent call last): + ... +TypeError: key must be an instance of str +>>> c.get_multi([1, 2]) +Traceback (most recent call last): + ... +TypeError: key must be an instance of str + +Also test some flush all. +>>> c.set("hi", "guys") +True +>>> c.get("hi") +'guys' +>>> c.flush_all() +True +>>> c.get("hi") +>>> + +Get and set booleans. Note that a boolean turns into an integer, mostly because +that's how python-memcached handles it. (Most likely because Python considers +the bool type to be an integer.) +>>> c.set("test", True) +True +>>> c.get("test") +1 +""" + +import _pylibmc +import socket + +test_server = (_pylibmc.server_type_tcp, "localhost", 11211) + +def get_version(addr): + (type_, host, port) = addr + if (type_ != _pylibmc.server_type_tcp): + raise NotImplementedError("test server can only be on tcp for now") + else: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((host, port)) + s.send("version\r\n") + version = s.recv(4096) + s.close() + if not version.startswith("VERSION ") or not version.endswith("\r\n"): + raise ValueError("unexpected version return: %r" % (version,)) + else: + version = version[8:-2] + return version + +def is_alive(addr): + try: + return bool(get_version(addr)) + except (ValueError, socket.error): + return False + +if __name__ == "__main__": + if not is_alive(test_server): + raise SystemExit("Test server (%r) not alive." % (test_server,)) + import doctest + doctest.testmod()