yannweb/rpnifs
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Implemented RPNIterExpr pickle/unpickle methods

Browse source
This commit is contained in:
Yann Weber 2023-09-18 00:19:31 +02:00
commit 28f0fdbb20
11 changed files with 439 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

247
python_if.c
View file

@ -243,9 +243,10 @@ of mmap.mmap");
}
expr_self->mmap = mmap_obj;
expr_self->_mmap = expr_self->mm_buff.buf;
// Creating rif with a new memory map
expr_self->rif = rpn_if_new(rif_params, expr_self->mm_buff.buf);
expr_self->rif = rpn_if_new(rif_params, expr_self->mm_buff.buf, NULL);
if(!expr_self->rif)
{
PyErr_Format(PyExc_RuntimeError,
@ -254,19 +255,7 @@ of mmap.mmap");
}
// Creating the tuple holding RPNExpr instances of expressions
expr_self->expr = PyTuple_New(expr_self->rif->params->rpn_sz);
for(size_t i=0; i<expr_self->rif->params->rpn_sz;i++)
{
rpn_expr_t *expr = &expr_self->rif->rpn[i];
PyObject *instance = rpnexpr_init_borrowing(expr);
if(!instance)
{
return -1;
}
PyTuple_SET_ITEM(expr_self->expr, i, instance);
}
return 0;
return _rpnif_init_expr_tuple(expr_self);
}
/**@brief Returns the parameters in a named tuple */
@ -295,10 +284,13 @@ PyObject *rpnif_set_mmap(PyObject *self, PyObject *mm_obj)
}
// Release old mmap
PyBuffer_Release(&expr_self->mm_buff);
Py_DECREF(expr_self->mmap);
if(expr_self->mmap)
{
PyBuffer_Release(&expr_self->mm_buff);
Py_DECREF(expr_self->mmap);
}
if(expr_self->rif->self_mem)
if(expr_self->rif->self_mem) // should never be true ?
{
const size_t mm_sz = expr_self->rif->params->mem_sz *\
expr_self->rif->params->value_sz;
@ -308,6 +300,7 @@ PyObject *rpnif_set_mmap(PyObject *self, PyObject *mm_obj)
// Set new mmap in python struct & rif struct
expr_self->mm_buff = buff;
expr_self->mmap = mm_obj;
expr_self->_mmap = buff.buf;
expr_self->rif->mem = buff.buf;
expr_self->rif->self_mem = 0;
@ -625,11 +618,14 @@ void rpnif_del(PyObject *self)
rpn_if_free(expr_self->rif);
expr_self->rif = NULL;
}
if(expr_self->mm_buff.buf)
if(expr_self->mmap)
{
PyBuffer_Release(&expr_self->mm_buff);
if(expr_self->mm_buff.buf)
{
PyBuffer_Release(&expr_self->mm_buff);
}
Py_DECREF(expr_self->mmap);
}
Py_DECREF(expr_self->mmap);
}
@ -842,7 +838,6 @@ int rpnif_getbuffer(PyObject *self, Py_buffer *view, int flags)
rpn_if_default_data_t *data = (rpn_if_default_data_t*)expr_self->rif->params->data;
view->buf = expr_self->rif->mem;
dprintf(2, "buffer addr %p\n", expr_self->mm_buff.buf);
view->obj = self;
view->len = expr_self->rif->params->mem_sz * expr_self->rif->params->value_sz;
view->readonly = 0;
@ -1111,19 +1106,194 @@ PyObject* rpnif_repr(PyObject *self)
PyObject* rpnif_getstate(PyObject *self, PyObject *noargs)
{
PyErr_SetString(PyExc_NotImplementedError,
"getstate Not implemented");
return NULL;
PyRPNIterExpr_t *expr_self = (PyRPNIterExpr_t*)self;
const rpn_if_param_t *params = expr_self->rif->params;
rpn_if_default_data_t *data = (rpn_if_default_data_t*)(params->data);
Py_RETURN_NONE;
const size_t const_val_sz = data->res_flag == RPN_IF_RES_CONST ? 1 : \
(data->res_flag == RPN_IF_RES_CONST_RGBA ? 4 : 0);
const size_t nszlim = data->ndim +
(data->pos_flag == RPN_IF_POSITION_XDIM ?1:0);
size_t buf_sz = sizeof(void*) + \
sizeof(rpn_if_param_t) + \
sizeof(rpn_if_default_data_t) + \
sizeof(rpn_value_t) * ( \
params->rpn_sz + params->rpn_argc) + \
sizeof(size_t) * nszlim + \
sizeof(rpn_value_t) * const_val_sz;
// all stack allocation grouped here
void *new_rpn[params->rpn_sz];
size_t rpn_sz[params->rpn_sz];
size_t sz_max = 0;
for(size_t i=0; i<params->rpn_sz; i++)
{
rpn_sz[i] = rpn_expr_serialize(&expr_self->rif->rpn[i],
NULL, 0);
// each expression stores a size and a picled repr
buf_sz += rpn_sz[i] + sizeof(size_t);
if(rpn_sz[i] > sz_max) { sz_max = rpn_sz[i]; }
}
void *buf = malloc(buf_sz);
if(!buf)
{
PyErr_Format(PyExc_MemoryError,
"Unable to allocate pickled representation : ",
strerror(errno));
return NULL;
}
bzero(buf, buf_sz);
bzero(new_rpn, sizeof(*new_rpn));
void **ptr = buf; // mmap ptr
rpn_if_param_t *new_params = (void*)(ptr+1);
rpn_if_default_data_t *new_data = (void*)(new_params + 1);
size_t *new_size_lim = (void*)(new_data+1);
rpn_value_t *new_const_val = (void*)(new_size_lim + nszlim);
size_t *new_rpn_sz = (void*)(new_const_val + const_val_sz);
size_t cur_offset = sizeof(size_t)*params->rpn_sz;
*ptr = expr_self->_mmap;
new_params->mem_sz = params->mem_sz;
new_params->value_sz = params->value_sz;
new_params->rpn_sz = params->rpn_sz;
new_params->rpn_argc = params->rpn_argc;
new_params->rpn_stack_sz = params->rpn_stack_sz;
// The following should be more efficient, but result
// in some badly initialized bytes in new_params struct
// padding...
//*new_params = *params;
new_params->getarg_f = NULL;
new_params->setres_f = NULL;
new_params->data = NULL;
new_data->pos_flag = data->pos_flag;
new_data->res_flag = data->res_flag;
new_data->ndim = data->ndim;
// Same as above
//*new_data = *data;
new_data->size_lim = NULL;
new_data->const_val = NULL;
memcpy(new_size_lim, data->size_lim, sizeof(size_t)*nszlim);
if(const_val_sz)
{
memcpy(new_const_val, data->const_val, sizeof(rpn_value_t)*const_val_sz);
}
// set sizes & rpn expressions
for(size_t i=0; i<params->rpn_sz; i++)
{
new_rpn[i] = ((void*)new_rpn_sz)+cur_offset;
new_rpn_sz[i] = rpn_sz[i];
bzero(new_rpn[i], rpn_sz[i]);
cur_offset += rpn_sz[i];
if(!rpn_expr_serialize(&expr_self->rif->rpn[i],
new_rpn[i], rpn_sz[i]))
{
goto err_rpn;
}
}
PyObject *res = PyBytes_FromStringAndSize(buf, buf_sz);
return res;
err_rpn:
/**@todo handle error */
free(buf);
return NULL;
}
PyObject* rpnif_setstate(PyObject *self, PyObject *state_bytes)
{
PyErr_SetString(PyExc_NotImplementedError,
"setstate Not implemented");
return NULL;
/**@todo TODO write the function */
PyRPNIterExpr_t *expr_self = (PyRPNIterExpr_t*)self;
if(!PyBytes_Check(state_bytes)) /* Arg check */
{
PyErr_SetString(PyExc_TypeError, "Expected bytes");
return NULL;
}
size_t bsize = PyBytes_GET_SIZE(state_bytes);
const void *data = (void*)PyBytes_AS_STRING(state_bytes);
if(bsize < sizeof(void*))
{
PyErr_SetString(PyExc_ValueError, "Smaller than expected given");
return NULL;
}
rpn_if_param_t *params, *ser_params;
rpn_if_default_data_t *ser_data;
size_t *ser_size_lim;
rpn_value_t *ser_const_val;
expr_self->mmap = NULL;
expr_self->_mmap = *(void**)data;
ser_params = (void*)((void**)data+1);
ser_data = (void*)(ser_params+1);
ser_size_lim = (void*)(ser_data+1);
const size_t nszlim = ser_data->ndim + \
(ser_data->pos_flag == RPN_IF_POSITION_XDIM ?1:0);
ser_const_val = (void*)(ser_size_lim + nszlim);
params = rpn_if_default_params(ser_data->pos_flag, ser_data->res_flag,
ser_size_lim, ser_const_val,
ser_params->rpn_stack_sz);
const size_t const_val_sz = ser_data->res_flag == RPN_IF_RES_CONST?1:\
(ser_data->res_flag == RPN_IF_RES_CONST_RGBA?4:0);
rpn_expr_t *ser_rpn = malloc(sizeof(*ser_rpn) * ser_params->rpn_sz);
if(!ser_rpn)
{
PyErr_Format(PyExc_MemoryError,
"Unable to alloc rpn expressions : %s",
strerror(errno));
return NULL;
}
size_t *ser_rpn_sz = (void*)(ser_const_val + const_val_sz);
void *ser_rpn_buf = (void*)(ser_rpn_sz + ser_params->rpn_sz);
for(size_t i=0; i<ser_params->rpn_sz; i++)
{
if(rpn_expr_deserialize(&ser_rpn[i], ser_rpn_buf, ser_rpn_sz[i]) < 0)
{
PyErr_Format(PyExc_ValueError,
"Unable to deserialize expr#%ld : %s",
i, ser_rpn[i].err_reason);
return NULL;
}
ser_rpn_buf = (void*)ser_rpn_buf + ser_rpn_sz[i];
}
expr_self->rif = rpn_if_new(params, expr_self->_mmap, ser_rpn);
if(!expr_self->rif)
{
PyErr_SetString(PyExc_ValueError,
"Unable to initialize rif expression");
return NULL;
}
expr_self->ndim = ser_data->ndim;
if(_rpnif_init_expr_tuple(expr_self) < 0)
{
return NULL;
}
Py_RETURN_NONE;
}
@ -1354,6 +1524,25 @@ PyObject *_rpnif_params_to_tuple(const rpn_if_param_t *_params)
return res;
}
int _rpnif_init_expr_tuple(PyRPNIterExpr_t *expr_self)
{
expr_self->expr = PyTuple_New(expr_self->rif->params->rpn_sz);
for(size_t i=0; i<expr_self->rif->params->rpn_sz;i++)
{
rpn_expr_t *expr = &expr_self->rif->rpn[i];
PyObject *instance = rpnexpr_init_borrowing(expr);
if(!instance)
{
//! @todo set exception
return -1;
}
PyTuple_SET_ITEM(expr_self->expr, i, instance);
}
return 0;
}
/**@def _ret_append(key)
* @hiderefs
* local macro */

25
python_if.h
View file

@ -61,6 +61,20 @@ extern PyObject *mmap_module;
extern PyObject *mmap_cls;
/**@brief Structure holding RPNIterExpr objects
*
* @warning Pickling methods are implemented for a limited usage.
* The mmap.mmap instance olded by the structure (sadly) cannot be
* pickled... In order to allow using RPNExprIter instances in
* multiprocessing stuffs, the piclking method will "export" only
* a pointer on the shared mmap. This implies that if the GC pops out
* on the mmap, the unpickled instance will segfault...
* This implies that unpickled instances has limited capabilities (no
* way to get back the mmap.mmap instance).
* There is a way (using another shared mmap at module level ?) to
* implement a custom reference count on the mmap pointer in order
* to determine when to DECREF the mmap.mmap instance.
*
* @todo think about implementing a safe pickle/unpickle method
* @ingroup pymod_pyrpn_RPNExprIter */
typedef struct
{
@ -76,12 +90,17 @@ typedef struct
/**@brief Python tuple with instances of RPNExpr */
PyObject *expr;
/**@brief Python mmap.mmap instance representing rif memory map */
/**@brief Python mmap.mmap instance representing rif memory map
* @note NULL if unpickled instance */
PyObject *mmap;
/**@brief Memory map buffer allowing acces to underlying pointer */
/**@brief Memory map buffer allowing acces to underlying pointer
* @note unrelevant if unpickled instance */
Py_buffer mm_buff;
/**@brief Memory map buffer pointer */
void *_mmap;
} PyRPNIterExpr_t;
/**@brief RPNIterExpr.params static method
@ -292,5 +311,7 @@ rpn_if_param_t *_rpnif_get_params(unsigned short pos_flag, unsigned short res_fl
*/
PyObject *_rpnif_params_to_tuple(const rpn_if_param_t *params);
int _rpnif_init_expr_tuple(PyRPNIterExpr_t *expr_self);
#endif

33
rpn_if.c
View file

@ -18,7 +18,8 @@
*/
#include "rpn_if.h"
rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap)
rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap,
rpn_expr_t *init_exprs)
{
rpn_if_t *res;
size_t i;
@ -74,6 +75,36 @@ rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap)
res->rpn_args = &(res->rpn_res[params->rpn_sz]);
if(init_exprs)
{ // using existing exppressions, checking them
short err = 0;
for(size_t i=0; i<params->rpn_sz; i++)
{
if(init_exprs[i].args_count != params->rpn_argc)
{
err = 1;
snprintf(init_exprs[i].err_reason, 128,
"Expected %ld arguments but expression got %ld",
params->rpn_argc,
init_exprs[i].args_count);
}
else if(init_exprs[i].stack_sz != params->rpn_stack_sz)
{
err = 1;
snprintf(init_exprs[i].err_reason, 128,
"Expected %d element stack but expression got %d",
params->rpn_stack_sz,
init_exprs[i].stack_sz);
}
}
if(err)
{
goto rpn_malloc_err;
}
res->rpn = init_exprs;
return res;
}
res->rpn = malloc(sizeof(rpn_expr_t) * params->rpn_sz);
if(!res->rpn)
{

5
rpn_if.h
View file

@ -118,9 +118,12 @@ struct rpn_if_s
/**@brief Alloc a new @ref rpn_if_s using given parameters
* @param params IF parameters
* @param memmap A suitable memory map. If NULL given, a new mmap is used
* @param init_exprs If no NULL uses this expression (steal refs), else uses
* new empty expressions
* @return A pointer on an allocated @ref rpn_if_s
*/
rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap);
rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap,
rpn_expr_t *init_exprs);
/**@brief Deallocate an @ref rpn_if_s and its ressources and close associated
* @ref rpn_expr_s

5
rpn_if_default.c
View file

@ -108,7 +108,10 @@ rpn_if_param_t* rpn_if_default_params(short pos_flag, short res_flag,
perror("Unable to alloc iterated function params");
return NULL;
}
res->data = data = (rpn_if_default_data_t*)(&(res[1]));
data = (rpn_if_default_data_t*)(res + 1);
bzero(res, sizeof(*res));
bzero(data, sizeof(*data));
res->data = data;
data->pos_flag = pos_flag;
data->res_flag = res_flag;

2
rpn_if_default.h
View file

@ -87,7 +87,7 @@ struct rpn_if_default_data_s
*/
size_t *size_lim;
/** Number of dimention (if XDIM ndim = len(size_lim)-1) */
/** Number of dimensions (if XDIM ndim = len(size_lim)-1) */
size_t ndim;
/**@brief Store constant values to set mem given res_flag

2
rpn_ifs.c
View file

@ -81,7 +81,7 @@ size_t rpn_ifs_add_if(rpn_ifs_t *rifs, unsigned int weight)
rifs->weight[rifs->if_sz] = weight;
//WRONG expr ARGUMENT !!!
rifs->rpn_if[rifs->if_sz] = rpn_if_new(&(rifs->params), rifs->mem);
rifs->rpn_if[rifs->if_sz] = rpn_if_new(&(rifs->params), rifs->mem, NULL);
if(!rifs->rpn_if[rifs->if_sz])
{
return 0;

4
tests/test_rpn.c
View file

@ -255,7 +255,7 @@ int test_rpn_if_default()
return -1;
}
rif = rpn_if_new(rif_param, NULL);
rif = rpn_if_new(rif_param, NULL, NULL);
if(!rif)
{
perror("rpn_if_new() failed");
@ -282,7 +282,7 @@ int test_rpn_if_default2()
return -1;
}
rif = rpn_if_new(rif_param, NULL);
rif = rpn_if_new(rif_param, NULL, NULL);
if(!rif)
{
perror("rpn_if_new() failed");

2
tests/tests_rpn_mutate.py
View file

@ -61,7 +61,7 @@ class TestRpnExprMutation(unittest.TestCase):
def test_random_int_mutation_params(self):
""" Testing int random values for parameters checking resulting expr len """
for _ in range(50):
m_params = [random.randint(0,20) for _ in range(4)]
m_params = [random.randint(1,20) for _ in range(4)]
expt_len = (m_params[0]-m_params[1])/sum(m_params)
expt_len = 0 if expt_len < 0 else expt_len
params = [1] + m_params + [(0,0,0),(0,0,0)]

31
tests/tests_rpniter.py
View file

@ -32,7 +32,7 @@ except (ImportError, NameError) as e:
file=sys.stderr)
raise e
from utils import Progress
from utils import *
class TestRpnExprCopy(unittest.TestCase):
@ -149,6 +149,35 @@ class TestRPNIterInit(unittest.TestCase):
rif = pyrpn.RPNIterExpr(*args)
self.assertEqual(rif.shape(), expt)
def test_pickling(self):
""" Test pickling/unpickling """
tests = [((pyrpn.const.POS_XY,
pyrpn.const.RESULT_COUNT,
(640,480)), (640,480)),
((pyrpn.const.POS_LINEAR,
pyrpn.const.RESULT_BOOL,
(1024,)), (1024,)),
((pyrpn.const.POS_LINEAR,
pyrpn.const.RESULT_RGBA,
(1024,)), (1024,4)),
((pyrpn.const.POS_XDIM,
pyrpn.const.RESULT_BOOL,
(2,640,480)), (640,480)),
((pyrpn.const.POS_XDIM,
pyrpn.const.RESULT_RGB,
(5,13,37,13,12,42)), (13,37,13,12,42,3)),
]
for args, expt in tests:
for _ in range(10):
rif = pyrpn.RPNIterExpr(*args)
for ex in rif.expressions:
ex.mutate(n_mutations = 15)
with self.subTest(rif=rif):
st = pickle.dumps(rif)
rif2 = pickle.loads(st)
st2 = pickle.dumps(rif2)
self.assertEqual(st, st2)
@skipIfNoNumpy()
def test_buffer(self):
""" Test the len on buffer interface using numpy """

123
tests/utils.py
View file

@ -17,6 +17,8 @@
# along with geneifs. if not, see <http://www.gnu.org/licenses/>.
#
from ctypes import *
import struct
import sys
import math
@ -78,4 +80,125 @@ class Progress:
raise expt
def RPNExpr_pickle_bin_dump(state):
""" Return a dict containing rpn state dump fields
"""
ser_fields = ['token_sz', 'stack_sz', 'argc', 'state']
ser_expr = struct.unpack('QQQQ', state[:32])
ser_expr = dict(zip(ser_fields, ser_expr))
err = state[32:32+128]
if err == b'\x00' * 128:
err = None
ser_expr['err_reason'] = err
token_sz = 24
tok_start = 32+128
tok_end = tok_start + (token_sz * ser_expr['token_sz'])
tokens = [struct.unpack('QQQ', state[tok_start+(24*i):tok_start+(24*(i+1))])
for i in range(ser_expr['token_sz'])]
tok_types = ['op', 'arg', 'val']
#pretty_toks=[{'type': tok_types[tok[0]],
# 'type_raw': tok[0],
# 'v1': hex(tok[1]),
# 'v2': hex(tok[2])}
# for tok in tokens]
pretty_toks = [(tok_types[tok[0]],)+tok for tok in tokens]
ser_expr['tokens'] = pretty_toks
#ser_expr['tokens'] = tokens
stack_start = tok_end
stack_end = stack_start + (8*ser_expr['stack_sz'])
if len(state) < stack_end:
print("ERROR expt size = %d but got %d" % (stack_end, len(state)))
ser_expr['stack'] = 'FAULT'
return ser_expr
stack = struct.unpack('L'*ser_expr['stack_sz'], state[stack_start:stack_end])
ser_expr['stack'] = stack
return ser_expr
def RPNIterExpr_pickle_bin_dump(dump):
import pprint
params_fields = ('mem_sz','value_sz','rpn_sz','rpn_argc','rpn_stack_sz',
'getarg_f','getres_f','data_ptr')
data_fields = ('pos_flag','res_flag', 'size_lim_ptr', 'ndim', 'const_val_ptr')
ptr = dump[0:8]
params = dump[8:72]
data = dump[72:104]
print("PARAMS : %r" % params)
print("DATA : %r" % data)
params = struct.unpack('LLLLbPPP', params)
params = dict(zip(params_fields, params))
data = struct.unpack('hhPLP', data)
data = dict(zip(data_fields, data))
params['data'] = data
sz_sz = data['ndim'] + (1 if data['pos_flag'] == pyrpn.const.POS_XDIM else 0)
sz_end = 104 + 8*sz_sz
sz_lim = struct.unpack('L' * sz_sz, dump[104:sz_end])
data['sz_lim'] = sz_lim
const_sz = 0
if data['res_flag'] == pyrpn.const.RESULT_CONST:
const_sz = 1
elif data['res_flag'] == pyrpn.const.RESULT_CONST_RGBA:
const_sz = 4
const_end = sz_end + (8*const_sz)
if const_sz:
const_val = struct.unpack('L'*const_sz, dump[sz_end:const_end])
else:
const_val = []
print("SZLIM : %r" % (dump[104:sz_end]))
print("CONST : %r" % (dump[sz_end:const_end]))
data['const_val'] = const_val
rpn_sz_end = const_end + (8*params['rpn_sz'])
rpn_sz = struct.unpack('L'*params['rpn_sz'], dump[const_end:rpn_sz_end])
ddump = {'params': params, 'rpn_sz': rpn_sz, 'expr': dict()}
rpn_buf_start = rpn_sz_end
for i in range(params['rpn_sz']):
rpn_buf_end = rpn_buf_start + rpn_sz[i]
rpn_state = dump[rpn_buf_start:rpn_buf_end]
print("RPN#%d[%d:%d] = %r" % (i, rpn_buf_start, rpn_buf_end, rpn_state))
try:
ddump['expr'][i] = RPNExpr_pickle_bin_dump(rpn_state)
except Exception as expt:
print("ERROR : ", expt)
rpn_buf_start = rpn_buf_end
return ddump
def bindump(data):
for i in range(0,len(data), 0x10):
print("%02X " % i, data[i:i+0x10])
if len(data) % 0x10:
last = ((len(data) / 0x10)+1)
print("%02X " % last, data[last:])