Implementing serialization in rpn_jit and use it in python_rpnexpr get/setstate
This commit is contained in:
parent
3eed2b7f3e
commit
7179fd5814
6 changed files with 199 additions and 185 deletions
200
python_rpnexpr.c
200
python_rpnexpr.c
|
|
@ -246,49 +246,16 @@ void rpnexpr_del(PyObject *self)
|
|||
|
||||
PyObject* rpnexpr_getexprstate(PyObject *self, PyObject *noargs)
|
||||
{
|
||||
/*
|
||||
PyObject *base64_mod, *base64_encode, *gzip, *compress;
|
||||
PyObject *bytes_repr, *comp, *res;
|
||||
if(!(base64_mod = PyImport_ImportModule("base64")))
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
if(!(base64_encode = PyObject_GetAttrString(base64_mod, "b64encode")))
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
if(!(gzip = PyImport_ImportModule("gzip")))
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
if(!(compress = PyObject_GetAttrString(gzip, "compress")))
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bytes_repr = _rpnexpr_getstate(self, noargs, 0);
|
||||
|
||||
res = PyObject_CallOneArg(base64_encode, bytes_repr);
|
||||
Py_DECREF(bytes_repr);
|
||||
return res;
|
||||
|
||||
comp = PyObject_CallOneArg(compress, bytes_repr);
|
||||
Py_DECREF(bytes_repr);
|
||||
|
||||
res = PyObject_CallOneArg(base64_encode, comp);
|
||||
Py_DECREF(comp);
|
||||
return res;
|
||||
*/
|
||||
/**@todo check if usefull */
|
||||
PyErr_SetString(PyExc_NotImplementedError, "Not implemented...");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
PyObject* rpnexpr_getstate(PyObject *self, PyObject *noargs)
|
||||
{
|
||||
PyObject *res, *part;
|
||||
PyRPNExpr_state_t resbuf;
|
||||
PyObject *res;
|
||||
PyRPNExpr_t *expr_self;
|
||||
size_t total_sz, i;
|
||||
size_t total_sz;
|
||||
char err_str[128];
|
||||
|
||||
expr_self = (PyRPNExpr_t*)self;
|
||||
|
|
@ -302,90 +269,26 @@ PyObject* rpnexpr_getstate(PyObject *self, PyObject *noargs)
|
|||
return NULL;
|
||||
}
|
||||
|
||||
total_sz = sizeof(PyRPNExpr_state_t);
|
||||
total_sz += expr_self->rpn->toks.tokens_sz * sizeof(rpn_token_t);
|
||||
total_sz += expr_self->rpn->stack_sz * sizeof(rpn_value_t);
|
||||
total_sz = rpn_expr_serialize(expr_self->rpn, NULL, 0);
|
||||
|
||||
bzero(&resbuf, sizeof(resbuf));
|
||||
resbuf.total_sz = total_sz;
|
||||
resbuf.argc = expr_self->rpn->args_count;
|
||||
resbuf.stack_sz = expr_self->rpn->stack_sz;
|
||||
resbuf.token_sz = expr_self->rpn->toks.tokens_sz;
|
||||
char serial[total_sz];
|
||||
|
||||
if(!(res = PyBytes_FromStringAndSize((char*)&resbuf, sizeof(resbuf))))
|
||||
rpn_expr_serialize(expr_self->rpn, serial, total_sz);
|
||||
if(!(res = PyBytes_FromStringAndSize(serial, total_sz)))
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
if(resbuf.stack_sz)
|
||||
{
|
||||
if(!(part=PyBytes_FromStringAndSize(
|
||||
(char*)expr_self->rpn->stack,
|
||||
sizeof(rpn_value_t) * resbuf.stack_sz)))
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
PyBytes_ConcatAndDel(&res, part);
|
||||
if(!res)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
if(resbuf.token_sz)
|
||||
{
|
||||
rpn_token_t tokens[resbuf.token_sz];
|
||||
bzero(tokens, sizeof(rpn_token_t) * resbuf.token_sz);
|
||||
|
||||
for(i=0; i<expr_self->rpn->toks.tokens_sz; i++)
|
||||
{
|
||||
const rpn_token_t *cur = &expr_self->rpn->toks.tokens[i];
|
||||
tokens[i].type = cur->type;
|
||||
switch(tokens[i].type)
|
||||
{
|
||||
case RPN_op:
|
||||
tokens[i].op_n = cur->op_n;
|
||||
tokens[i].op = NULL;
|
||||
break;
|
||||
case RPN_arg:
|
||||
tokens[i].arg_n = cur->arg_n;
|
||||
break;
|
||||
case RPN_val:
|
||||
tokens[i].value = cur->value;
|
||||
break;
|
||||
default:
|
||||
PyErr_SetString(PyExc_RuntimeError,
|
||||
"__getstate__ invalid token");
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
if(!(part=PyBytes_FromStringAndSize(
|
||||
(char*)tokens,
|
||||
sizeof(rpn_token_t) * resbuf.token_sz)))
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
PyBytes_ConcatAndDel(&res, part);
|
||||
if(!res)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
PyObject* rpnexpr_setstate(PyObject *self, PyObject *state_bytes)
|
||||
{
|
||||
PyObject *tmp, *tmp2;
|
||||
PyRPNExpr_state_t *state;
|
||||
PyRPNExpr_t *expr_self;
|
||||
rpn_value_t *stack;
|
||||
rpn_tokenized_t toks;
|
||||
const char *data;
|
||||
size_t bsize, csize;
|
||||
size_t bsize;
|
||||
int err;
|
||||
char err_str[256];
|
||||
size_t i;
|
||||
|
||||
expr_self = (PyRPNExpr_t*)self;
|
||||
|
||||
|
|
@ -424,37 +327,14 @@ should not be initialized");
|
|||
/* Checking data size */
|
||||
bsize = PyBytes_GET_SIZE(state_bytes);
|
||||
data = PyBytes_AS_STRING(state_bytes);
|
||||
state = (PyRPNExpr_state_t*)data;
|
||||
|
||||
if(bsize < sizeof(size_t))
|
||||
{
|
||||
PyErr_SetString(PyExc_ValueError, "Invalid argument");
|
||||
}
|
||||
if(bsize != state->total_sz)
|
||||
{
|
||||
snprintf(err_str, 128,
|
||||
"RPNExpr.__setstate__() error : expected state to \
|
||||
contains %ld bytes but %ld found",
|
||||
state->total_sz, bsize);
|
||||
PyErr_SetString(PyExc_ValueError, err_str);
|
||||
}
|
||||
|
||||
/* Checking data "integrity" */
|
||||
csize = sizeof(PyRPNExpr_state_t) +
|
||||
state->token_sz * sizeof(rpn_token_t) +
|
||||
state->stack_sz * sizeof(rpn_value_t);
|
||||
if(state->total_sz != csize)
|
||||
{
|
||||
snprintf(err_str, 128,
|
||||
"RPNExpr.__setstate__() error : should have %ld as \
|
||||
total size but %ld found",
|
||||
csize, state->total_sz);
|
||||
PyErr_SetString(PyExc_ValueError, err_str);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
/* Alloc and init rpn expression & args buffer*/
|
||||
//alloc & init
|
||||
if(!(expr_self->rpn = malloc(sizeof(rpn_expr_t))))
|
||||
{
|
||||
err = errno;
|
||||
|
|
@ -466,7 +346,16 @@ rpn_expr_t : %s",
|
|||
return NULL;
|
||||
}
|
||||
bzero(expr_self->rpn, sizeof(rpn_expr_t));
|
||||
if(!(expr_self->args = malloc(sizeof(rpn_value_t)*state->argc)))
|
||||
|
||||
if(rpn_expr_deserialize(expr_self->rpn, data, bsize) < 0)
|
||||
{
|
||||
PyErr_Format(PyExc_ValueError,
|
||||
"RPNExpr.__setstate__() fails to deserialize (%s)) %s",
|
||||
strerror(errno), expr_self->rpn->err_reason);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if(!(expr_self->args = malloc(sizeof(rpn_value_t)*expr_self->rpn->args_count)))
|
||||
{
|
||||
err = errno;
|
||||
snprintf(err_str, 128,
|
||||
|
|
@ -477,58 +366,7 @@ args buffer : %s",
|
|||
return NULL;
|
||||
}
|
||||
|
||||
if(rpn_expr_init(expr_self->rpn, state->stack_sz, state->argc) < 0)
|
||||
{
|
||||
snprintf(err_str, 256,
|
||||
"RPNExpr.__setstate__() failed to init RPN expression \
|
||||
: %s",
|
||||
expr_self->rpn->err_reason);
|
||||
PyErr_SetString(PyExc_RuntimeError, err_str);
|
||||
goto free_err;
|
||||
}
|
||||
|
||||
/* restore stack & untokenize expression */
|
||||
stack = (rpn_value_t*)(state+1);
|
||||
memcpy(expr_self->rpn->stack, stack,
|
||||
sizeof(rpn_value_t)*state->stack_sz);
|
||||
|
||||
toks.argc = state->argc;
|
||||
toks.tokens_sz = state->token_sz;
|
||||
toks.tokens = malloc(sizeof(rpn_token_t)*state->token_sz);
|
||||
if(!toks.tokens)
|
||||
{
|
||||
snprintf(err_str, 128,
|
||||
"RPNExpr.__setstate__() failed to allocate tokens \
|
||||
: %s",
|
||||
strerror(errno));
|
||||
PyErr_SetString(PyExc_RuntimeError, err_str);
|
||||
goto close_err;
|
||||
}
|
||||
memcpy(toks.tokens, (rpn_token_t*)(stack+state->stack_sz),
|
||||
sizeof(rpn_token_t)*state->token_sz);
|
||||
|
||||
for(i=0; i<toks.tokens_sz;i++)
|
||||
{
|
||||
// restore op pointers
|
||||
toks.tokens[i].op = &(rpn_ops[toks.tokens[i].op_n]);
|
||||
}
|
||||
|
||||
if(rpn_expr_untokenize(expr_self->rpn, &toks, 0) < 0)
|
||||
{
|
||||
snprintf(err_str, 256,
|
||||
"RPNExpr.__setstate__() unable to untokenize : %s",
|
||||
expr_self->rpn->err_reason);
|
||||
PyErr_SetString(PyExc_RuntimeError, err_str);
|
||||
goto close_err;
|
||||
}
|
||||
expr_self->rpn->toks = toks;
|
||||
|
||||
|
||||
Py_RETURN_NONE;
|
||||
close_err:
|
||||
rpn_expr_close(expr_self->rpn);
|
||||
free_err:
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
156
rpn_jit.c
156
rpn_jit.c
|
|
@ -139,7 +139,7 @@ int rpn_expr_compile(rpn_expr_t *expr, const char *code)
|
|||
return _rpn_expr_compile_expr(expr);
|
||||
}
|
||||
|
||||
int rpn_expr_untokenize(rpn_expr_t *expr, rpn_tokenized_t *tokens, char long_op)
|
||||
int rpn_expr_untokenize(rpn_expr_t *expr, const rpn_tokenized_t *tokens, char long_op)
|
||||
{
|
||||
int err;
|
||||
size_t i;
|
||||
|
|
@ -298,6 +298,160 @@ char* rpn_random(size_t op_sz, size_t args_count)
|
|||
return buff;
|
||||
}
|
||||
|
||||
size_t rpn_expr_serialize(rpn_expr_t* expr, void *buf, size_t buf_sz)
|
||||
{
|
||||
const size_t total_sz = sizeof(rpn_expr_serial_t) + \
|
||||
(sizeof(rpn_token_t)*expr->toks.tokens_sz) + \
|
||||
(sizeof(rpn_value_t)*expr->stack_sz);
|
||||
|
||||
if(total_sz < buf_sz || !buf)
|
||||
{
|
||||
return total_sz;
|
||||
}
|
||||
|
||||
bzero(buf, total_sz);
|
||||
|
||||
rpn_expr_serial_t *ser = buf;
|
||||
rpn_token_t *tokens = (void*)buf + sizeof(*ser);
|
||||
rpn_value_t *stack = (void*)(tokens + expr->toks.tokens_sz);
|
||||
|
||||
ser->token_sz = expr->toks.tokens_sz;
|
||||
ser->stack_sz = expr->stack_sz;
|
||||
ser->argc = expr->toks.argc;
|
||||
ser->state = expr->state;
|
||||
memcpy(ser->err_reason, expr->err_reason, sizeof(ser->err_reason));
|
||||
memcpy(stack, expr->stack, expr->stack_sz);
|
||||
|
||||
for(size_t i=0; i < ser->token_sz; i++)
|
||||
{
|
||||
tokens[i].type = expr->toks.tokens[i].type;
|
||||
switch(expr->toks.tokens[i].type)
|
||||
{
|
||||
case RPN_arg:
|
||||
tokens[i].arg_n = expr->toks.tokens[i].arg_n;
|
||||
break;
|
||||
case RPN_val:
|
||||
tokens[i].value = expr->toks.tokens[i].value;
|
||||
break;
|
||||
case RPN_op:
|
||||
tokens[i].op_n = expr->toks.tokens[i].op_n;
|
||||
break;
|
||||
default:
|
||||
// SHOULD NEVER APPEND !
|
||||
errno = EINVAL;
|
||||
goto err;
|
||||
}
|
||||
}
|
||||
|
||||
return total_sz;
|
||||
|
||||
err:
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int rpn_expr_deserialize(rpn_expr_t* expr, const void *buf, size_t buf_sz)
|
||||
{
|
||||
int err = EINVAL;
|
||||
const rpn_expr_serial_t *ser = buf;
|
||||
|
||||
if(!expr)
|
||||
{
|
||||
errno = EINVAL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if(buf_sz < sizeof(rpn_expr_serial_t))
|
||||
{
|
||||
snprintf(expr->err_reason, 128,
|
||||
"Given buffer is to small (%ld bytes) to deserialize", buf_sz);
|
||||
err = EINVAL;
|
||||
goto err;
|
||||
}
|
||||
|
||||
const size_t total_sz = sizeof(rpn_expr_serial_t) + \
|
||||
(sizeof(rpn_token_t)*ser->token_sz) + \
|
||||
(sizeof(rpn_value_t)*ser->stack_sz);
|
||||
if(buf_sz < total_sz)
|
||||
{
|
||||
snprintf(expr->err_reason, 128,
|
||||
"Expected %ld bytes but %ld given",
|
||||
total_sz, buf_sz);
|
||||
err = EINVAL;
|
||||
goto err;
|
||||
}
|
||||
|
||||
rpn_token_t *tokens = (void*)buf + sizeof(*ser);
|
||||
rpn_value_t *stack = (void*)(tokens + ser->token_sz);
|
||||
|
||||
if(rpn_expr_init(expr, ser->stack_sz, ser->argc) < 0)
|
||||
{
|
||||
err = EINVAL;
|
||||
goto err;
|
||||
}
|
||||
|
||||
expr->state = ser->state;
|
||||
memcpy(expr->err_reason, ser->err_reason, 128);
|
||||
memcpy(expr->stack, stack, sizeof(rpn_value_t)*ser->stack_sz);
|
||||
|
||||
expr->args_count = expr->toks.argc = ser->argc;
|
||||
|
||||
rpn_tokenized_t toks;
|
||||
toks.argc = ser->argc;
|
||||
toks.tokens_sz = ser->token_sz;
|
||||
toks.tokens = malloc(sizeof(rpn_token_t)*ser->token_sz);
|
||||
if(!toks.tokens)
|
||||
{
|
||||
err = errno;
|
||||
snprintf(expr->err_reason, 128,
|
||||
"Unable to allocate tokens : %s",
|
||||
strerror(errno));
|
||||
goto err_expr;
|
||||
}
|
||||
|
||||
|
||||
for(size_t i=0; i < ser->token_sz; i++)
|
||||
{
|
||||
toks.tokens[i].type = tokens[i].type;
|
||||
switch(tokens[i].type)
|
||||
{
|
||||
case RPN_val:
|
||||
toks.tokens[i].value = tokens[i].value;
|
||||
break;
|
||||
case RPN_arg:
|
||||
toks.tokens[i].arg_n = tokens[i].arg_n;
|
||||
break;
|
||||
case RPN_op:
|
||||
toks.tokens[i].op_n = tokens[i].op_n;
|
||||
toks.tokens[i].op = &(rpn_ops[tokens[i].op_n]);
|
||||
break;
|
||||
default:
|
||||
snprintf(expr->err_reason, 128,
|
||||
"Invalid token type encountered %d", tokens[i].type);
|
||||
err = EINVAL;
|
||||
goto err_expr;
|
||||
}
|
||||
}
|
||||
|
||||
if(rpn_expr_untokenize(expr, &toks, 0) < 0)
|
||||
{
|
||||
err = EINVAL;
|
||||
goto err_expr;
|
||||
}
|
||||
|
||||
expr->toks = toks;
|
||||
|
||||
return 0;
|
||||
|
||||
err_expr:
|
||||
rpn_expr_close(expr);
|
||||
err:
|
||||
expr->state = RPN_ERROR;
|
||||
errno = err;
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
int _rpn_expr_compile_expr(rpn_expr_t* expr)
|
||||
{
|
||||
rpn_tokenizer_t tokenizer;
|
||||
|
|
|
|||
22
rpn_jit.h
22
rpn_jit.h
|
|
@ -100,6 +100,9 @@ typedef unsigned long (*rpn_run_f)(unsigned long, unsigned long*, void*);
|
|||
* @ingroup rpn */
|
||||
typedef struct rpn_expr_s rpn_expr_t;
|
||||
|
||||
/** @brief Serialized expression form */
|
||||
typedef struct rpn_expr_serial_s rpn_expr_serial_t;
|
||||
|
||||
/**@brief Stores RPN expression informations
|
||||
* @ingroup rpn
|
||||
*/
|
||||
|
|
@ -134,6 +137,18 @@ struct rpn_expr_s
|
|||
char err_reason[128];
|
||||
};
|
||||
|
||||
/**@brief Serialized form of an RPN expression
|
||||
* @ingroup rpn
|
||||
*/
|
||||
struct rpn_expr_serial_s
|
||||
{
|
||||
size_t token_sz;
|
||||
unsigned char stack_sz;
|
||||
size_t argc;
|
||||
short state;
|
||||
char err_reason[128];
|
||||
};
|
||||
|
||||
/**@brief Initialize a new @ref rpn_expr_s
|
||||
* @param expr Pointer on the expression
|
||||
* @param stack_sz Expression stack size
|
||||
|
|
@ -188,7 +203,7 @@ int rpn_expr_compile(rpn_expr_t *expr, const char *code);
|
|||
* can be reused)
|
||||
* @ingroup rpn
|
||||
*/
|
||||
int rpn_expr_untokenize(rpn_expr_t *expr, rpn_tokenized_t *tokens, char long_op);
|
||||
int rpn_expr_untokenize(rpn_expr_t *expr, const rpn_tokenized_t *tokens, char long_op);
|
||||
|
||||
/**@brief Generate a new random rpn_expression
|
||||
* @param op_sz Number of token in generated expression
|
||||
|
|
@ -233,6 +248,11 @@ void rpn_expr_close(rpn_expr_t* expr);
|
|||
*/
|
||||
void rpn_expr_reset_stack(rpn_expr_t *expr);
|
||||
|
||||
/**@todo document*/
|
||||
size_t rpn_expr_serialize(rpn_expr_t* expr, void *buf, size_t buf_sz);
|
||||
/**@todo document*/
|
||||
int rpn_expr_deserialize(rpn_expr_t* expr, const void *buf, size_t buf_sz);
|
||||
|
||||
/**@brief Copy precompiled code from @ref rpn_lib.h in expression code map
|
||||
* @param expr The expression being compiled
|
||||
* @param token Pointer on token informations
|
||||
|
|
|
|||
|
|
@ -214,7 +214,7 @@ void rpn_tokenizer_free(rpn_tokenizer_t *tokenizer)
|
|||
}
|
||||
}
|
||||
|
||||
char* rpn_tokenized_expr(rpn_tokenized_t *tokens, char long_op)
|
||||
char* rpn_tokenized_expr(const rpn_tokenized_t *tokens, char long_op)
|
||||
{
|
||||
size_t expr_sz, i;
|
||||
int err, written;
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ int rpn_tokenize(const char *token, rpn_token_t *dst, char error[64]);
|
|||
* @return A newly allocated char* that should be deallocated using free()
|
||||
* @ingroup rpn_tokenize
|
||||
*/
|
||||
char* rpn_tokenized_expr(rpn_tokenized_t *tokens, char long_op);
|
||||
char* rpn_tokenized_expr(const rpn_tokenized_t *tokens, char long_op);
|
||||
|
||||
/**@brief Returns NULL or a pointer on corresponding operation infos
|
||||
* @param token The token we want to match
|
||||
|
|
|
|||
|
|
@ -92,8 +92,10 @@ class TestRpnCompile(unittest.TestCase):
|
|||
pik = pickle.dumps(expr)
|
||||
new_expr = pickle.loads(pik)
|
||||
self.assertEqual(str(expr), str(new_expr))
|
||||
pik2 = pickle.dumps(new_expr)
|
||||
args = [random.randint(0,IMAX) for _ in range(argc)]
|
||||
self.assertEqual(expr.eval(*args), new_expr.eval(*args))
|
||||
self.assertEqual(pik, pik2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue