rays
/
pylibmc
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' of git://github.com/lericson/pylibmc

master
John Watson 2010-10-09 11:31:25 -07:00
parent ddd2f011f7 e3a6828569
commit c6cb0db9bc
11 changed files with 547 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
_pylibmcmodule.c
View File

@ -308,7 +308,9 @@ error:
static PyObject *_PylibMC_parse_memcached_value(char *value, size_t size,
uint32_t flags) {
PyObject *retval, *tmp;
PyObject *retval = NULL;
PyObject *tmp = NULL;
uint32_t dtype = flags & PYLIBMC_FLAG_TYPES;
#if USE_ZLIB
PyObject *inflated = NULL;
@ -327,22 +329,26 @@ static PyObject *_PylibMC_parse_memcached_value(char *value, size_t size,
}
#endif
retval = NULL;
switch (flags & PYLIBMC_FLAG_TYPES) {
switch (dtype) {
case PYLIBMC_FLAG_PICKLE:
retval = _PylibMC_Unpickle(value, size);
break;
case PYLIBMC_FLAG_INTEGER:
case PYLIBMC_FLAG_LONG:
retval = PyInt_FromString(value, NULL, 10);
break;
case PYLIBMC_FLAG_BOOL:
if ((tmp = PyInt_FromString(value, NULL, 10)) == NULL) {
return NULL;
/* PyInt_FromString doesn't take a length param and we're
not NULL-terminated, so we'll have to make an
intermediate Python string out of it */
tmp = PyString_FromStringAndSize(value, size);
if(tmp == NULL) {
goto cleanup;
}
retval = PyInt_FromString(PyString_AS_STRING(tmp), NULL, 10);
if(retval != NULL && dtype == PYLIBMC_FLAG_BOOL) {
Py_DECREF(tmp);
tmp = retval;
retval = PyBool_FromLong(PyInt_AS_LONG(tmp));
}
retval = PyBool_FromLong(PyInt_AS_LONG(tmp));
Py_DECREF(tmp);
break;
case PYLIBMC_FLAG_NONE:
retval = PyString_FromStringAndSize(value, (Py_ssize_t)size);
@ -352,10 +358,14 @@ static PyObject *_PylibMC_parse_memcached_value(char *value, size_t size,
"unknown memcached key flags %u", flags);
}
cleanup:
#if USE_ZLIB
Py_XDECREF(inflated);
#endif
Py_XDECREF(tmp);
return retval;
}
@ -394,6 +404,72 @@ static PyObject *PylibMC_Client_get(PylibMC_Client *self, PyObject *arg) {
return PylibMC_ErrFromMemcached(self, "memcached_get", error);
}
static PyObject *PylibMC_Client_gets(PylibMC_Client *self, PyObject *arg) {
const char* keys[2];
size_t keylengths[2];
memcached_result_st results_obj;
memcached_result_st *results = NULL;
memcached_return_t rc;
PyObject* ret = NULL;
if (!_PylibMC_CheckKey(arg)) {
return NULL;
} else if (!PySequence_Length(arg)) {
/* Others do this, so... */
Py_RETURN_NONE;
} else if (!memcached_behavior_get(self->mc, MEMCACHED_BEHAVIOR_SUPPORT_CAS)) {
PyErr_SetString(PyExc_ValueError, "gets without cas behavior");
return NULL;
}
/* Use an mget to fetch the key.
* mget is the only function that returns a memcached_result_st,
* which is the only way to get at the returned cas value. */
*keys = PyString_AS_STRING(arg);
*keylengths = (size_t)PyString_GET_SIZE(arg);
Py_BEGIN_ALLOW_THREADS;
rc = memcached_mget(self->mc, keys, keylengths, 1);
if (rc == MEMCACHED_SUCCESS) {
memcached_result_create(self->mc, &results_obj);
/* this will be NULL if the key wasn't found, or
memcached_result_st if it was */
results = memcached_fetch_result(self->mc, &results_obj, &rc);
}
Py_END_ALLOW_THREADS;
if (rc == MEMCACHED_SUCCESS && results != NULL) {
const char *mc_val = memcached_result_value(results);
size_t val_size = memcached_result_length(results);
uint32_t flags = memcached_result_flags(results);
uint64_t cas = memcached_result_cas(results);
ret = _PylibMC_parse_memcached_value((char *)mc_val, val_size, flags);
ret = Py_BuildValue("(NL)", ret, cas);
/* we have to fetch the last result from the mget cursor */
memcached_fetch_result(self->mc, &results_obj, &rc);
if (rc != MEMCACHED_END) {
Py_DECREF(ret);
ret = NULL;
PyErr_SetString(PyExc_RuntimeError, "fetch not done");
}
} else if (rc == MEMCACHED_END) {
/* Key not found => (None, None) */
ret = Py_BuildValue("(OO)", Py_None, Py_None);
} else {
ret = PylibMC_ErrFromMemcached(self, "memcached_gets", rc);
}
/* deallocate any indirect buffers, even though the struct itself
is on our stack */
memcached_result_free(&results_obj);
return ret;
}
/* {{{ Set commands (set, replace, add, prepend, append) */
static PyObject *_PylibMC_RunSetCommandSingle(PylibMC_Client *self,
_PylibMC_SetCommand f, char *fname, PyObject *args,
@ -477,8 +553,6 @@ static PyObject *_PylibMC_RunSetCommandMulti(PylibMC_Client* self,
goto cleanup;
}
memset((void *)serialized, 0x0, nkeys * sizeof(pylibmc_mset));
/**
* We're pointing into existing Python memory with the 'key' members of
* pylibmc_mset (extracted using PyDict_Next) and during
@ -544,6 +618,65 @@ cleanup:
return retval;
}
static PyObject *_PylibMC_RunCasCommand(PylibMC_Client *self,
PyObject *args, PyObject *kwds) {
/* function called by the set/add/etc commands */
static char *kws[] = { "key", "val", "cas", "time", NULL };
PyObject *ret = NULL;
PyObject *key;
PyObject *value;
uint64_t cas = 0;
unsigned int time = 0; /* this will be turned into a time_t */
bool success = false;
memcached_return rc;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "SOL|I", kws,
&key, &value, &cas,
&time)) {
return NULL;
}
if (!memcached_behavior_get(self->mc, MEMCACHED_BEHAVIOR_SUPPORT_CAS)) {
PyErr_SetString(PyExc_ValueError, "cas without cas behavior");
return NULL;
}
pylibmc_mset mset = { NULL };
/* TODO: because it's RunSetCommand that does the zlib
compression, cas can't currently use compressed values. */
success = _PylibMC_SerializeValue(key, NULL, value, time, &mset);
if (!success || PyErr_Occurred() != NULL) {
goto cleanup;
}
Py_BEGIN_ALLOW_THREADS;
rc = memcached_cas(self->mc,
mset.key, mset.key_len,
mset.value, mset.value_len,
mset.time, mset.flags, cas);
Py_END_ALLOW_THREADS;
switch(rc) {
case MEMCACHED_SUCCESS:
Py_INCREF(Py_True);
ret = Py_True;
break;
case MEMCACHED_DATA_EXISTS:
Py_INCREF(Py_False);
ret = Py_False;
break;
default:
PylibMC_ErrFromMemcached(self, "memcached_cas", rc);
}
cleanup:
_PylibMC_FreeMset(&mset);
return ret;
}
static void _PylibMC_FreeMset(pylibmc_mset *mset) {
Py_XDECREF(mset->key_obj);
mset->key_obj = NULL;
@ -561,6 +694,9 @@ static int _PylibMC_SerializeValue(PyObject* key_obj,
PyObject* value_obj,
time_t time,
pylibmc_mset* serialized) {
/* first zero the whole structure out */
memset((void *)serialized, 0x0, sizeof(pylibmc_mset));
serialized->time = time;
serialized->success = false;
serialized->flags = PYLIBMC_FLAG_NONE;
@ -650,9 +786,9 @@ static int _PylibMC_SerializeValue(PyObject* key_obj,
return false;
}
if(PyString_AsStringAndSize(store_val, &serialized->value,
&serialized->value_len) == -1) {
if(serialized->flags == PYLIBMC_FLAG_NONE) {
if (PyString_AsStringAndSize(store_val, &serialized->value,
&serialized->value_len) == -1) {
if (serialized->flags == PYLIBMC_FLAG_NONE) {
/* For some reason we weren't able to extract the value/size
from a string that we were explicitly passed, that we
INCREF'd above */
@ -662,7 +798,7 @@ static int _PylibMC_SerializeValue(PyObject* key_obj,
}
/* So now we have a reference to a string that we may have
created. we need that to keep existing while we release the HIL,
created. we need that to keep existing while we release the GIL,
so we need to hold the reference, but we need to free it up when
we're done */
serialized->value_obj = store_val;
@ -791,6 +927,11 @@ static PyObject *PylibMC_Client_append(PylibMC_Client *self, PyObject *args,
}
/* }}} */
static PyObject *PylibMC_Client_cas(PylibMC_Client *self, PyObject *args,
PyObject *kwds) {
return _PylibMC_RunCasCommand(self, args, kwds);
}
static PyObject *PylibMC_Client_delete(PylibMC_Client *self, PyObject *args) {
PyObject *retval;
char *key;
@ -984,8 +1125,6 @@ cleanup:
return retval;
}
static PyObject *PylibMC_Client_incr(PylibMC_Client *self, PyObject *args) {
return _PylibMC_IncrSingle(self, memcached_increment, args);
}

6
_pylibmcmodule.h
View File

@ -238,11 +238,13 @@ static PylibMC_Client *PylibMC_ClientType_new(PyTypeObject *, PyObject *,
static void PylibMC_ClientType_dealloc(PylibMC_Client *);
static int PylibMC_Client_init(PylibMC_Client *, PyObject *, PyObject *);
static PyObject *PylibMC_Client_get(PylibMC_Client *, PyObject *arg);
static PyObject *PylibMC_Client_gets(PylibMC_Client *, PyObject *arg);
static PyObject *PylibMC_Client_set(PylibMC_Client *, PyObject *, PyObject *);
static PyObject *PylibMC_Client_replace(PylibMC_Client *, PyObject *, PyObject *);
static PyObject *PylibMC_Client_add(PylibMC_Client *, PyObject *, PyObject *);
static PyObject *PylibMC_Client_prepend(PylibMC_Client *, PyObject *, PyObject *);
static PyObject *PylibMC_Client_append(PylibMC_Client *, PyObject *, PyObject *);
static PyObject *PylibMC_Client_cas(PylibMC_Client *, PyObject *, PyObject *);
static PyObject *PylibMC_Client_delete(PylibMC_Client *, PyObject *);
static PyObject *PylibMC_Client_incr(PylibMC_Client *, PyObject *);
static PyObject *PylibMC_Client_decr(PylibMC_Client *, PyObject *);
@ -287,6 +289,8 @@ static bool _PylibMC_IncrDecr(PylibMC_Client*, pylibmc_incr*, size_t);
static PyMethodDef PylibMC_ClientType_methods[] = {
{"get", (PyCFunction)PylibMC_Client_get, METH_O,
"Retrieve a key from a memcached."},
{"gets", (PyCFunction)PylibMC_Client_gets, METH_O,
"Retrieve a key and cas_id from a memcached."},
{"set", (PyCFunction)PylibMC_Client_set, METH_VARARGS|METH_KEYWORDS,
"Set a key unconditionally."},
{"replace", (PyCFunction)PylibMC_Client_replace, METH_VARARGS|METH_KEYWORDS,
@ -297,6 +301,8 @@ static PyMethodDef PylibMC_ClientType_methods[] = {
"Prepend data to a key."},
{"append", (PyCFunction)PylibMC_Client_append, METH_VARARGS|METH_KEYWORDS,
"Append data to a key."},
{"cas", (PyCFunction)PylibMC_Client_cas, METH_VARARGS|METH_KEYWORDS,
"Attempt to compare-and-store a key by CAS ID."},
{"delete", (PyCFunction)PylibMC_Client_delete, METH_VARARGS,
"Delete a key."},
{"incr", (PyCFunction)PylibMC_Client_incr, METH_VARARGS,

8
coders.rst 100644
View File

@ -0,0 +1,8 @@
The List of Honored Men
=======================
* Ludvig Ericson <ludvig@lericson.se>
* ketralnis <ketralnis@reddit.com>
* Ruda Moura <ruda.moura@corp.terra.com.br>
* Noah Silas <noah@mahalo.com>
* Johan Bergström <johan@bergstroem.nu>

307
pylibmc.py
View File

@ -1,307 +0,0 @@
"""Snappy libmemcached wrapper
pylibmc is a Python wrapper around TangentOrg's libmemcached library.
The interface is intentionally made as close to python-memcached as possible,
so that applications can drop-in replace it.
Example usage
=============
Create a connection and configure it::
>>> import pylibmc
>>> mc = pylibmc.Client(["127.0.0.1"], binary=True)
>>> mc.behaviors = {"tcp_nodelay": True, "ketama": True}
Basic operation::
>>> mc.set("some_key", "Some value")
True
>>> value = mc.get("some_key")
>>> value
'Some value'
>>> mc.set("another_key", 3)
True
>>> mc.delete("another_key")
True
>>> mc.set("key", "1") # str or int is fine
True
Atomic increments and decrements::
>>> mc.incr("key")
2L
>>> mc.decr("key")
1L
Batch operation::
>>> mc.get_multi(["key", "another_key"])
{'key': '1'}
>>> mc.set_multi({"cats": ["on acid", "furry"], "dogs": True})
[]
>>> mc.get_multi(["cats", "dogs"])
{'cats': ['on acid', 'furry'], 'dogs': True}
>>> mc.delete_multi(["cats", "dogs", "nonextant"])
False
>>> mc.add_multi({"cats": ["on acid", "furry"], "dogs": True})
[]
>>> mc.get_multi(["cats", "dogs"])
{'cats': ['on acid', 'furry'], 'dogs': True}
>>> mc.add_multi({"cats": "not set", "dogs": "definitely not set", "bacon": "yummy"})
['cats', 'dogs']
>>> mc.get_multi(["cats", "dogs", "bacon"])
{'cats': ['on acid', 'furry'], 'bacon': 'yummy', 'dogs': True}
>>> mc.delete_multi(["cats", "dogs", "bacon"])
True
Further Reading
===============
See http://sendapatch.se/projects/pylibmc/
"""
from __future__ import with_statement
import _pylibmc
from Queue import Queue
__all__ = ["hashers", "distributions", "Client"]
__version__ = _pylibmc.__version__
support_compression = _pylibmc.support_compression
errors = tuple(e for (n, e) in _pylibmc.exceptions)
# *Cough* Uhm, not the prettiest of things but this unpacks all exception
# objects and sets them on the very module object currently constructed.
import sys
modself = sys.modules[__name__]
for name, exc in _pylibmc.exceptions:
setattr(modself, name, exc)
hashers, hashers_rvs = {}, {}
distributions, distributions_rvs = {}, {}
# Not the prettiest way of doing things, but works well.
for name in dir(_pylibmc):
if name.startswith("hash_"):
key, value = name[5:], getattr(_pylibmc, name)
hashers[key] = value
hashers_rvs[value] = key
elif name.startswith("distribution_"):
key, value = name[13:].replace("_", " "), getattr(_pylibmc, name)
distributions[key] = value
distributions_rvs[value] = key
class BehaviorDict(dict):
def __init__(self, client, *args, **kwds):
super(BehaviorDict, self).__init__(*args, **kwds)
self.client = client
def __setitem__(self, name, value):
super(BehaviorDict, self).__setitem__(name, value)
self.client.set_behaviors({name: value})
def update(self, d):
super(BehaviorDict, self).update(d)
self.client.set_behaviors(d.copy())
class Client(_pylibmc.client):
def __init__(self, servers, binary=False):
"""Initialize a memcached client instance.
This connects to the servers in *servers*, which will default to being
TCP servers. If it looks like a filesystem path, a UNIX socket. If
prefixed with `udp:`, a UDP connection.
If *binary* is True, the binary memcached protocol is used.
"""
self.binary = binary
self.addresses = list(servers)
addr_tups = []
for server in servers:
addr = server
port = 11211
if server.startswith("udp:"):
stype = _pylibmc.server_type_udp
addr = addr[4:]
if ":" in server:
(addr, port) = addr.split(":", 1)
port = int(port)
elif ":" in server:
stype = _pylibmc.server_type_tcp
(addr, port) = server.split(":", 1)
port = int(port)
elif "/" in server:
stype = _pylibmc.server_type_unix
port = 0
else:
stype = _pylibmc.server_type_tcp
addr_tups.append((stype, addr, port))
super(Client, self).__init__(servers=addr_tups, binary=binary)
def __repr__(self):
return "%s(%r, binary=%r)" % (self.__class__.__name__,
self.addresses, self.binary)
def __str__(self):
addrs = ", ".join(map(str, self.addresses))
return "<%s for %s, binary=%r>" % (self.__class__.__name__,
addrs, self.binary)
def get_behaviors(self):
"""Gets the behaviors from the underlying C client instance.
Reverses the integer constants for `hash` and `distribution` into more
understandable string values. See *set_behaviors* for info.
"""
bvrs = super(Client, self).get_behaviors()
bvrs["hash"] = hashers_rvs[bvrs["hash"]]
bvrs["distribution"] = distributions_rvs[bvrs["distribution"]]
return BehaviorDict(self, bvrs)
def set_behaviors(self, behaviors):
"""Sets the behaviors on the underlying C client instance.
Takes care of morphing the `hash` key, if specified, into the
corresponding integer constant (which the C client expects.) If,
however, an unknown value is specified, it's passed on to the C client
(where it most surely will error out.)
This also happens for `distribution`.
"""
behaviors = behaviors.copy()
if behaviors.get("hash") is not None:
behaviors["hash"] = hashers[behaviors["hash"]]
if behaviors.get("ketama_hash") is not None:
behaviors["ketama_hash"] = hashers[behaviors["ketama_hash"]]
if behaviors.get("distribution") is not None:
behaviors["distribution"] = distributions[behaviors["distribution"]]
return super(Client, self).set_behaviors(behaviors)
behaviors = property(get_behaviors, set_behaviors)
@property
def behaviours(self):
raise AttributeError("nobody uses british spellings")
def clone(self):
obj = super(Client, self).clone()
obj.addresses = list(self.addresses)
obj.binary = self.binary
return obj
from contextlib import contextmanager
class ClientPool(Queue):
"""Client pooling helper.
This is mostly useful in threaded environments, because a client isn't
thread-safe at all. Instead, what you want to do is have each thread use
its own client, but you don't want to reconnect these all the time.
The solution is a pool, and this class is a helper for that.
>>> mc = Client(["127.0.0.1"])
>>> pool = ClientPool()
>>> pool.fill(mc, 4)
>>> with pool.reserve() as mc:
... mc.set("hi", "ho")
... mc.delete("hi")
...
True
True
"""
def __init__(self, mc=None, n_slots=None):
Queue.__init__(self, n_slots)
if mc is not None:
self.fill(mc, n_slots)
@contextmanager
def reserve(self, timeout=None):
"""Context manager for reserving a client from the pool.
If *timeout* is given, it specifiecs how long to wait for a client to
become available.
"""
mc = self.get(True, timeout=timeout)
try:
yield mc
finally:
self.put(mc)
def fill(self, mc, n_slots):
"""Fill *n_slots* of the pool with clones of *mc*."""
for i in xrange(n_slots):
self.put(mc.clone())
class ThreadMappedPool(dict):
"""Much like the *ClientPool*, helps you with pooling.
In a threaded environment, you'd most likely want to have a client per
thread. And there'd be no harm in one thread keeping the same client at all
times. So, why not map threads to clients? That's what this class does.
If a client is reserved, this class checks for a key based on the current
thread, and if none exists, clones the master client and inserts that key.
>>> mc = Client(["127.0.0.1"])
>>> pool = ThreadMappedPool(mc)
>>> with pool.reserve() as mc:
... mc.set("hi", "ho")
... mc.delete("hi")
...
True
True
"""
def __new__(cls, master):
return super(ThreadMappedPool, cls).__new__(cls)
def __init__(self, master):
self.master = master
@contextmanager
def reserve(self):
"""Reserve a client.
Creates a new client based on the master client if none exists for the
current thread.
"""
key = thread.get_ident()
mc = self.pop(key, None)
if mc is None:
mc = self.master.clone()
try:
yield mc
finally:
self[key] = mc
# This makes sure ThreadMappedPool doesn't exist with non-thread Pythons.
try:
import thread
except ImportError:
del ThreadMappedPool
if __name__ == "__main__":
import sys
import code
svr_addrs = []
sys.stderr.write("pylibmc interactive shell\n\n")
sys.stderr.write("Input list of servers, end by a blank line\n")
binary = False
if sys.argv[1:] == ["--binary"]:
binary = True
while True:
in_addr = raw_input("Address: ")
if not in_addr:
break
if not svr_addrs:
svr_addrs.append("127.0.0.1")
mc = Client(svr_addrs, binary=binary)
code.interact(banner="\nmc client available as `mc`", local={"mc": mc})

73
pylibmc/__init__.py 100644
View File

@ -0,0 +1,73 @@
"""Snappy libmemcached wrapper
pylibmc is a Python wrapper around TangentOrg's libmemcached library.
The interface is intentionally made as close to python-memcached as possible,
so that applications can drop-in replace it.
Example usage
=============
Create a connection and configure it::
>>> import pylibmc
>>> mc = pylibmc.Client(["127.0.0.1"], binary=True)
>>> mc.behaviors = {"tcp_nodelay": True, "ketama": True}
Basic operation::
>>> mc.set("some_key", "Some value")
True
>>> value = mc.get("some_key")
>>> value
'Some value'
>>> mc.set("another_key", 3)
True
>>> mc.delete("another_key")
True
>>> mc.set("key", "1") # str or int is fine
True
Atomic increments and decrements::
>>> mc.incr("key")
2L
>>> mc.decr("key")
1L
Batch operation::
>>> mc.get_multi(["key", "another_key"])
{'key': '1'}
>>> mc.set_multi({"cats": ["on acid", "furry"], "dogs": True})
[]
>>> mc.get_multi(["cats", "dogs"])
{'cats': ['on acid', 'furry'], 'dogs': True}
>>> mc.delete_multi(["cats", "dogs", "nonextant"])
False
>>> mc.add_multi({"cats": ["on acid", "furry"], "dogs": True})
[]
>>> mc.get_multi(["cats", "dogs"])
{'cats': ['on acid', 'furry'], 'dogs': True}
>>> mc.add_multi({"cats": "not set", "dogs": "definitely not set", "bacon": "yummy"})
['cats', 'dogs']
>>> mc.get_multi(["cats", "dogs", "bacon"])
{'cats': ['on acid', 'furry'], 'bacon': 'yummy', 'dogs': True}
>>> mc.delete_multi(["cats", "dogs", "bacon"])
True
Further Reading
===============
See http://sendapatch.se/projects/pylibmc/
"""
import _pylibmc
from .consts import hashers, distributions
from .client import Client
from .pools import ClientPool, ThreadMappedPool
support_compression = _pylibmc.support_compression
__version__ = _pylibmc.__version__
__all__ = ["hashers", "distributions", "Client",
"ClientPool", "ThreadMappedPool"]

48
pylibmc/__main__.py 100644
View File

@ -0,0 +1,48 @@
"""Interactive shell"""
import sys
import code
import random
import pylibmc
tips = [
"Want to use 127.0.0.1? Just hit Enter immediately.",
"This was supposed to be a list of tips but I...",
"I don't really know what to write here.",
"Really, hit Enter immediately and you'll connect to 127.0.0.1.",
"Did you know there's a --binary flag? Try it!",
"Want to use binary mode? Pass --binary as a sole argument."
]
def print_header(outf=sys.stdout):
outf.write("pylibmc interactive shell\n\n")
outf.write("Input list of servers, terminating by a blank line.\n")
outf.write(random.choice(tips) + "\n")
def collect_servers():
in_addr = raw_input("Address [127.0.0.1]: ")
while in_addr:
in_addr = raw_input("Address [<stop>]: ")
if in_addr:
yield in_addr
else:
yield "127.0.0.1"
banner = ("\nmc client available as `mc`\n"
"memface available as mf")
def interact(servers, banner=banner, binary=False):
mc = pylibmc.Client(servers, binary=binary)
mf = pylibmc.Memface(servers, binary=binary)
local = {"pylibmc": pylibmc,
"mf": mf, "mc": mc}
code.interact(banner=banner, local=local)
def main():
binary = False
if sys.argv[1:] == ["--binary"]:
binary = True
print_header()
interact(list(collect_servers()), binary=binary)
if __name__ == "__main__":
main()

90
pylibmc/client.py 100644
View File

@ -0,0 +1,90 @@
"""Python-level wrapper client"""
import _pylibmc
from .consts import (hashers, distributions,
hashers_rvs, distributions_rvs,
BehaviorDict)
class Client(_pylibmc.client):
def __init__(self, servers, binary=False):
"""Initialize a memcached client instance.
This connects to the servers in *servers*, which will default to being
TCP servers. If it looks like a filesystem path, a UNIX socket. If
prefixed with `udp:`, a UDP connection.
If *binary* is True, the binary memcached protocol is used.
"""
self.binary = binary
self.addresses = list(servers)
addr_tups = []
for server in servers:
addr = server
port = 11211
if server.startswith("udp:"):
stype = _pylibmc.server_type_udp
addr = addr[4:]
if ":" in server:
(addr, port) = addr.split(":", 1)
port = int(port)
elif ":" in server:
stype = _pylibmc.server_type_tcp
(addr, port) = server.split(":", 1)
port = int(port)
elif "/" in server:
stype = _pylibmc.server_type_unix
port = 0
else:
stype = _pylibmc.server_type_tcp
addr_tups.append((stype, addr, port))
super(Client, self).__init__(servers=addr_tups, binary=binary)
def __repr__(self):
return "%s(%r, binary=%r)" % (self.__class__.__name__,
self.addresses, self.binary)
def __str__(self):
addrs = ", ".join(map(str, self.addresses))
return "<%s for %s, binary=%r>" % (self.__class__.__name__,
addrs, self.binary)
def get_behaviors(self):
"""Gets the behaviors from the underlying C client instance.
Reverses the integer constants for `hash` and `distribution` into more
understandable string values. See *set_behaviors* for info.
"""
bvrs = super(Client, self).get_behaviors()
bvrs["hash"] = hashers_rvs[bvrs["hash"]]
bvrs["distribution"] = distributions_rvs[bvrs["distribution"]]
return BehaviorDict(self, bvrs)
def set_behaviors(self, behaviors):
"""Sets the behaviors on the underlying C client instance.
Takes care of morphing the `hash` key, if specified, into the
corresponding integer constant (which the C client expects.) If,
however, an unknown value is specified, it's passed on to the C client
(where it most surely will error out.)
This also happens for `distribution`.
"""
behaviors = behaviors.copy()
if behaviors.get("hash") is not None:
behaviors["hash"] = hashers[behaviors["hash"]]
if behaviors.get("ketama_hash") is not None:
behaviors["ketama_hash"] = hashers[behaviors["ketama_hash"]]
if behaviors.get("distribution") is not None:
behaviors["distribution"] = distributions[behaviors["distribution"]]
return super(Client, self).set_behaviors(behaviors)
behaviors = property(get_behaviors, set_behaviors)
@property
def behaviours(self):
raise AttributeError("nobody uses british spellings")
def clone(self):
obj = super(Client, self).clone()
obj.addresses = list(self.addresses)
obj.binary = self.binary
return obj

40
pylibmc/consts.py 100644
View File

@ -0,0 +1,40 @@
"""Constants and functionality related to them"""
import _pylibmc
#: Mapping of
errors = tuple(e for (n, e) in _pylibmc.exceptions)
# *Cough* Uhm, not the prettiest of things but this unpacks all exception
# objects and sets them on the package module object currently constructed.
import sys
modpkg = sys.modules[__package__]
modself = sys.modules[__package__]
for name, exc in _pylibmc.exceptions:
setattr(modself, name, exc)
setattr(modpkg, name, exc)
hashers, hashers_rvs = {}, {}
distributions, distributions_rvs = {}, {}
# Not the prettiest way of doing things, but works well.
for name in dir(_pylibmc):
if name.startswith("hash_"):
key, value = name[5:], getattr(_pylibmc, name)
hashers[key] = value
hashers_rvs[value] = key
elif name.startswith("distribution_"):
key, value = name[13:].replace("_", " "), getattr(_pylibmc, name)
distributions[key] = value
distributions_rvs[value] = key
class BehaviorDict(dict):
def __init__(self, client, *args, **kwds):
super(BehaviorDict, self).__init__(*args, **kwds)
self.client = client
def __setitem__(self, name, value):
super(BehaviorDict, self).__setitem__(name, value)
self.client.set_behaviors({name: value})
def update(self, d):
super(BehaviorDict, self).update(d)
self.client.set_behaviors(d.copy())

97
pylibmc/pools.py 100644
View File

@ -0,0 +1,97 @@
"""Pooling"""
from __future__ import with_statement
from contextlib import contextmanager
from Queue import Queue
class ClientPool(Queue):
"""Client pooling helper.
This is mostly useful in threaded environments, because a client isn't
thread-safe at all. Instead, what you want to do is have each thread use
its own client, but you don't want to reconnect these all the time.
The solution is a pool, and this class is a helper for that.
>>> mc = Client(["127.0.0.1"])
>>> pool = ClientPool()
>>> pool.fill(mc, 4)
>>> with pool.reserve() as mc:
... mc.set("hi", "ho")
... mc.delete("hi")
...
True
True
"""
def __init__(self, mc=None, n_slots=None):
Queue.__init__(self, n_slots)
if mc is not None:
self.fill(mc, n_slots)
@contextmanager
def reserve(self, timeout=None):
"""Context manager for reserving a client from the pool.
If *timeout* is given, it specifiecs how long to wait for a client to
become available.
"""
mc = self.get(True, timeout=timeout)
try:
yield mc
finally:
self.put(mc)
def fill(self, mc, n_slots):
"""Fill *n_slots* of the pool with clones of *mc*."""
for i in xrange(n_slots):
self.put(mc.clone())
class ThreadMappedPool(dict):
"""Much like the *ClientPool*, helps you with pooling.
In a threaded environment, you'd most likely want to have a client per
thread. And there'd be no harm in one thread keeping the same client at all
times. So, why not map threads to clients? That's what this class does.
If a client is reserved, this class checks for a key based on the current
thread, and if none exists, clones the master client and inserts that key.
>>> mc = Client(["127.0.0.1"])
>>> pool = ThreadMappedPool(mc)
>>> with pool.reserve() as mc:
... mc.set("hi", "ho")
... mc.delete("hi")
...
True
True
"""
def __new__(cls, master):
return super(ThreadMappedPool, cls).__new__(cls)
def __init__(self, master):
self.master = master
@contextmanager
def reserve(self):
"""Reserve a client.
Creates a new client based on the master client if none exists for the
current thread.
"""
key = thread.get_ident()
mc = self.pop(key, None)
if mc is None:
mc = self.master.clone()
try:
yield mc
finally:
self[key] = mc
# This makes sure ThreadMappedPool doesn't exist with non-thread Pythons.
try:
import thread
except ImportError:
ThreadMappedPool = None

2
setup.py
View File

@ -74,4 +74,4 @@ setup(name="pylibmc", version=version,
license="3-clause BSD <http://www.opensource.org/licenses/bsd-license.php>",
description="Quick and small memcached client for Python",
long_description=readme_text,
ext_modules=[pylibmc_ext], py_modules=["pylibmc"])
ext_modules=[pylibmc_ext], packages=["pylibmc"])

27
tests.py
View File

@ -263,6 +263,33 @@ Test server/client max length
Traceback (most recent call last):
...
ValueError: key too long, max is 250
Test CAS
>>> mc.behaviors['cas'] = True
>>> mc.delete('foo') and False
False
>>> mc.gets('foo')
(None, None)
>>> mc.set('foo', 'bar')
True
>>> foostr, cas = mc.gets('foo')
>>> foostr
'bar'
>>> mc.cas('foo', 'quux', cas+1)
False
>>> mc.cas('foo', 'baz', cas)
True
>>> mc.get('foo')
'baz'
>>> mc.behaviors['cas'] = False
>>> mc.gets('foo')
Traceback (most recent call last):
...
ValueError: gets without cas behavior
>>> mc.cas('foo', 'bar', 1)
Traceback (most recent call last):
...
ValueError: cas without cas behavior
"""
# Used to test pickling.