Bladeren bron

Implemented RPNIterExpr pickle/unpickle methods

Yann Weber 1 jaar geleden
bovenliggende
commit
28f0fdbb20
11 gewijzigde bestanden met toevoegingen van 439 en 40 verwijderingen
  1. 218
    29
      python_if.c
  2. 23
    2
      python_if.h
  3. 32
    1
      rpn_if.c
  4. 4
    1
      rpn_if.h
  5. 4
    1
      rpn_if_default.c
  6. 1
    1
      rpn_if_default.h
  7. 1
    1
      rpn_ifs.c
  8. 2
    2
      tests/test_rpn.c
  9. 1
    1
      tests/tests_rpn_mutate.py
  10. 30
    1
      tests/tests_rpniter.py
  11. 123
    0
      tests/utils.py

+ 218
- 29
python_if.c Bestand weergeven

@@ -243,9 +243,10 @@ of mmap.mmap");
243 243
 	}
244 244
 
245 245
 	expr_self->mmap = mmap_obj;
246
+	expr_self->_mmap = expr_self->mm_buff.buf;
246 247
 
247 248
 	// Creating rif with a new memory map
248
-	expr_self->rif = rpn_if_new(rif_params, expr_self->mm_buff.buf);
249
+	expr_self->rif = rpn_if_new(rif_params, expr_self->mm_buff.buf, NULL);
249 250
 	if(!expr_self->rif)
250 251
 	{
251 252
 		PyErr_Format(PyExc_RuntimeError,
@@ -254,19 +255,7 @@ of mmap.mmap");
254 255
 	}
255 256
 
256 257
 	// Creating the tuple holding RPNExpr instances of expressions
257
-	expr_self->expr = PyTuple_New(expr_self->rif->params->rpn_sz);
258
-	for(size_t i=0; i<expr_self->rif->params->rpn_sz;i++)
259
-	{
260
-		rpn_expr_t *expr = &expr_self->rif->rpn[i];
261
-		PyObject *instance = rpnexpr_init_borrowing(expr);
262
-		if(!instance)
263
-		{
264
-			return -1;
265
-		}
266
-		PyTuple_SET_ITEM(expr_self->expr, i, instance);
267
-	}
268
-
269
-	return 0;
258
+	return _rpnif_init_expr_tuple(expr_self);
270 259
 }
271 260
 
272 261
 /**@brief Returns the parameters in a named tuple */
@@ -295,10 +284,13 @@ PyObject *rpnif_set_mmap(PyObject *self, PyObject *mm_obj)
295 284
 	}
296 285
 
297 286
 	// Release old mmap
298
-	PyBuffer_Release(&expr_self->mm_buff);
299
-	Py_DECREF(expr_self->mmap);
287
+	if(expr_self->mmap)
288
+	{
289
+		PyBuffer_Release(&expr_self->mm_buff);
290
+		Py_DECREF(expr_self->mmap);
291
+	}
300 292
 
301
-	if(expr_self->rif->self_mem)
293
+	if(expr_self->rif->self_mem) // should never be true ?
302 294
 	{
303 295
 		const size_t mm_sz = expr_self->rif->params->mem_sz *\
304 296
 				     expr_self->rif->params->value_sz;
@@ -308,6 +300,7 @@ PyObject *rpnif_set_mmap(PyObject *self, PyObject *mm_obj)
308 300
 	// Set new mmap in python struct & rif struct
309 301
 	expr_self->mm_buff = buff;
310 302
 	expr_self->mmap = mm_obj;
303
+	expr_self->_mmap = buff.buf;
311 304
 
312 305
 	expr_self->rif->mem = buff.buf;
313 306
 	expr_self->rif->self_mem = 0;
@@ -625,11 +618,14 @@ void rpnif_del(PyObject *self)
625 618
 		rpn_if_free(expr_self->rif);
626 619
 		expr_self->rif = NULL;
627 620
 	}
628
-	if(expr_self->mm_buff.buf)
621
+	if(expr_self->mmap)
629 622
 	{
630
-		PyBuffer_Release(&expr_self->mm_buff);
623
+		if(expr_self->mm_buff.buf)
624
+		{
625
+			PyBuffer_Release(&expr_self->mm_buff);
626
+		}
627
+		Py_DECREF(expr_self->mmap);
631 628
 	}
632
-	Py_DECREF(expr_self->mmap);
633 629
 }
634 630
 
635 631
 
@@ -842,7 +838,6 @@ int rpnif_getbuffer(PyObject *self, Py_buffer *view, int flags)
842 838
 	rpn_if_default_data_t *data = (rpn_if_default_data_t*)expr_self->rif->params->data;
843 839
 
844 840
 	view->buf = expr_self->rif->mem;
845
-dprintf(2, "buffer addr %p\n", expr_self->mm_buff.buf);
846 841
 	view->obj = self;
847 842
 	view->len = expr_self->rif->params->mem_sz * expr_self->rif->params->value_sz;
848 843
 	view->readonly = 0;
@@ -1111,19 +1106,194 @@ PyObject* rpnif_repr(PyObject *self)
1111 1106
 
1112 1107
 PyObject* rpnif_getstate(PyObject *self, PyObject *noargs)
1113 1108
 {
1114
-	PyErr_SetString(PyExc_NotImplementedError,
1115
-		"getstate Not implemented");
1116
-	return NULL;
1109
+	PyRPNIterExpr_t *expr_self = (PyRPNIterExpr_t*)self;
1110
+	const rpn_if_param_t *params = expr_self->rif->params;
1111
+	rpn_if_default_data_t *data = (rpn_if_default_data_t*)(params->data);
1117 1112
 
1118
-	Py_RETURN_NONE;
1113
+	const size_t const_val_sz = data->res_flag == RPN_IF_RES_CONST ? 1 : \
1114
+			     (data->res_flag == RPN_IF_RES_CONST_RGBA ? 4 : 0);
1115
+	
1116
+	const size_t nszlim = data->ndim +
1117
+		(data->pos_flag == RPN_IF_POSITION_XDIM ?1:0);
1118
+
1119
+
1120
+	size_t buf_sz = sizeof(void*) + \
1121
+			sizeof(rpn_if_param_t) + \
1122
+			sizeof(rpn_if_default_data_t) + \
1123
+			sizeof(rpn_value_t) * ( \
1124
+					params->rpn_sz + params->rpn_argc) + \
1125
+			sizeof(size_t) * nszlim + \
1126
+			sizeof(rpn_value_t) * const_val_sz;
1127
+
1128
+	// all stack allocation grouped here
1129
+	void *new_rpn[params->rpn_sz];
1130
+
1131
+	size_t rpn_sz[params->rpn_sz];
1132
+
1133
+	size_t sz_max = 0;
1134
+	for(size_t i=0;  i<params->rpn_sz; i++)
1135
+	{
1136
+		rpn_sz[i] = rpn_expr_serialize(&expr_self->rif->rpn[i],
1137
+				NULL, 0);
1138
+		// each expression stores a size and a picled repr
1139
+		buf_sz += rpn_sz[i] + sizeof(size_t);
1140
+		if(rpn_sz[i] > sz_max) { sz_max = rpn_sz[i]; }
1141
+	}
1142
+
1143
+	void *buf = malloc(buf_sz);
1144
+	if(!buf)
1145
+	{
1146
+		PyErr_Format(PyExc_MemoryError,
1147
+				"Unable to allocate pickled representation : ",
1148
+				strerror(errno));
1149
+		return NULL;
1150
+	}
1151
+
1152
+	bzero(buf, buf_sz);
1153
+	bzero(new_rpn, sizeof(*new_rpn));
1154
+
1155
+	void **ptr = buf; // mmap ptr
1156
+	rpn_if_param_t *new_params = (void*)(ptr+1);
1157
+	rpn_if_default_data_t *new_data = (void*)(new_params + 1);
1158
+	size_t *new_size_lim = (void*)(new_data+1);
1159
+	rpn_value_t *new_const_val = (void*)(new_size_lim + nszlim);
1160
+	size_t *new_rpn_sz = (void*)(new_const_val + const_val_sz);
1161
+	size_t cur_offset = sizeof(size_t)*params->rpn_sz;
1162
+
1163
+	*ptr = expr_self->_mmap;
1164
+
1165
+	new_params->mem_sz = params->mem_sz;
1166
+	new_params->value_sz = params->value_sz;
1167
+	new_params->rpn_sz = params->rpn_sz;
1168
+	new_params->rpn_argc = params->rpn_argc;
1169
+	new_params->rpn_stack_sz = params->rpn_stack_sz;
1170
+	// The following should be more efficient, but result
1171
+	// in some badly initialized bytes in new_params struct
1172
+	// padding...
1173
+	//*new_params = *params;
1174
+
1175
+	new_params->getarg_f = NULL;
1176
+	new_params->setres_f = NULL;
1177
+	new_params->data = NULL;
1178
+
1179
+	new_data->pos_flag = data->pos_flag;
1180
+	new_data->res_flag = data->res_flag;
1181
+	new_data->ndim = data->ndim;
1182
+	// Same as above
1183
+	//*new_data = *data;
1184
+
1185
+	new_data->size_lim = NULL;
1186
+	new_data->const_val = NULL;
1187
+
1188
+	memcpy(new_size_lim, data->size_lim, sizeof(size_t)*nszlim);
1189
+	if(const_val_sz)
1190
+	{
1191
+		memcpy(new_const_val, data->const_val, sizeof(rpn_value_t)*const_val_sz);
1192
+	}
1193
+
1194
+	// set sizes & rpn expressions
1195
+	for(size_t i=0; i<params->rpn_sz; i++)
1196
+	{
1197
+		new_rpn[i] = ((void*)new_rpn_sz)+cur_offset;
1198
+		new_rpn_sz[i] = rpn_sz[i];
1199
+		bzero(new_rpn[i], rpn_sz[i]);
1200
+		cur_offset += rpn_sz[i];
1201
+		if(!rpn_expr_serialize(&expr_self->rif->rpn[i],
1202
+					new_rpn[i], rpn_sz[i]))
1203
+		{
1204
+			goto err_rpn;
1205
+		}
1206
+	}
1207
+
1208
+	PyObject *res = PyBytes_FromStringAndSize(buf, buf_sz);
1209
+	return res;
1210
+
1211
+err_rpn:
1212
+	/**@todo handle error */
1213
+	free(buf);
1214
+	return NULL;
1119 1215
 }
1120 1216
 
1121 1217
 PyObject* rpnif_setstate(PyObject *self, PyObject *state_bytes)
1122 1218
 {
1123
-	PyErr_SetString(PyExc_NotImplementedError,
1124
-		"setstate Not implemented");
1125
-	return NULL;
1126
-	/**@todo TODO write the function */
1219
+	PyRPNIterExpr_t *expr_self = (PyRPNIterExpr_t*)self;
1220
+
1221
+	if(!PyBytes_Check(state_bytes)) /* Arg check */
1222
+	{
1223
+		PyErr_SetString(PyExc_TypeError, "Expected bytes");
1224
+		return NULL;
1225
+	}
1226
+
1227
+	size_t bsize = PyBytes_GET_SIZE(state_bytes);
1228
+	const void *data = (void*)PyBytes_AS_STRING(state_bytes);
1229
+
1230
+	if(bsize < sizeof(void*))
1231
+	{
1232
+		PyErr_SetString(PyExc_ValueError, "Smaller than expected given");
1233
+		return NULL;
1234
+	}
1235
+
1236
+	rpn_if_param_t *params, *ser_params;
1237
+	rpn_if_default_data_t *ser_data;
1238
+	size_t *ser_size_lim;
1239
+	rpn_value_t *ser_const_val;
1240
+
1241
+	expr_self->mmap = NULL;
1242
+	expr_self->_mmap = *(void**)data;
1243
+
1244
+	ser_params = (void*)((void**)data+1);
1245
+	ser_data = (void*)(ser_params+1);
1246
+	ser_size_lim = (void*)(ser_data+1);
1247
+
1248
+	const size_t nszlim = ser_data->ndim + \
1249
+		(ser_data->pos_flag == RPN_IF_POSITION_XDIM ?1:0);
1250
+
1251
+	ser_const_val = (void*)(ser_size_lim + nszlim);
1252
+
1253
+	params = rpn_if_default_params(ser_data->pos_flag, ser_data->res_flag,
1254
+			ser_size_lim, ser_const_val,
1255
+			ser_params->rpn_stack_sz);
1256
+
1257
+	const size_t const_val_sz = ser_data->res_flag == RPN_IF_RES_CONST?1:\
1258
+			     (ser_data->res_flag == RPN_IF_RES_CONST_RGBA?4:0);
1259
+	
1260
+	rpn_expr_t *ser_rpn = malloc(sizeof(*ser_rpn) * ser_params->rpn_sz);
1261
+	if(!ser_rpn)
1262
+	{
1263
+		PyErr_Format(PyExc_MemoryError,
1264
+				"Unable to alloc rpn expressions : %s",
1265
+				strerror(errno));
1266
+		return NULL;
1267
+	}
1268
+	size_t *ser_rpn_sz = (void*)(ser_const_val + const_val_sz);
1269
+	void *ser_rpn_buf = (void*)(ser_rpn_sz + ser_params->rpn_sz);
1270
+
1271
+
1272
+	for(size_t i=0; i<ser_params->rpn_sz; i++)
1273
+	{
1274
+		if(rpn_expr_deserialize(&ser_rpn[i], ser_rpn_buf, ser_rpn_sz[i]) < 0)
1275
+		{
1276
+			PyErr_Format(PyExc_ValueError,
1277
+					"Unable to deserialize expr#%ld : %s",
1278
+					i, ser_rpn[i].err_reason);
1279
+			return NULL;
1280
+		}
1281
+		ser_rpn_buf = (void*)ser_rpn_buf + ser_rpn_sz[i];
1282
+	}
1283
+
1284
+	expr_self->rif = rpn_if_new(params, expr_self->_mmap, ser_rpn);
1285
+	if(!expr_self->rif)
1286
+	{
1287
+		PyErr_SetString(PyExc_ValueError,
1288
+				"Unable to initialize rif expression");
1289
+		return NULL;
1290
+	}
1291
+	expr_self->ndim = ser_data->ndim;
1292
+	if(_rpnif_init_expr_tuple(expr_self) < 0)
1293
+	{
1294
+		return NULL;
1295
+	}
1296
+
1127 1297
 	Py_RETURN_NONE;
1128 1298
 }
1129 1299
 
@@ -1354,6 +1524,25 @@ PyObject *_rpnif_params_to_tuple(const rpn_if_param_t *_params)
1354 1524
 	return res;
1355 1525
 }
1356 1526
 
1527
+
1528
+int _rpnif_init_expr_tuple(PyRPNIterExpr_t *expr_self)
1529
+{
1530
+	expr_self->expr = PyTuple_New(expr_self->rif->params->rpn_sz);
1531
+	for(size_t i=0; i<expr_self->rif->params->rpn_sz;i++)
1532
+	{
1533
+		rpn_expr_t *expr = &expr_self->rif->rpn[i];
1534
+		PyObject *instance = rpnexpr_init_borrowing(expr);
1535
+		if(!instance)
1536
+		{
1537
+			//! @todo set exception
1538
+			return -1;
1539
+		}
1540
+		PyTuple_SET_ITEM(expr_self->expr, i, instance);
1541
+	}
1542
+	return 0;
1543
+}
1544
+
1545
+
1357 1546
 /**@def _ret_append(key) 
1358 1547
  * @hiderefs
1359 1548
  * local macro */

+ 23
- 2
python_if.h Bestand weergeven

@@ -61,6 +61,20 @@ extern PyObject *mmap_module;
61 61
 extern PyObject *mmap_cls;
62 62
 
63 63
 /**@brief Structure holding RPNIterExpr objects
64
+ *
65
+ * @warning Pickling methods are implemented for a limited usage.
66
+ * The mmap.mmap instance olded by the structure (sadly) cannot be
67
+ * pickled... In order to allow using RPNExprIter instances in
68
+ * multiprocessing stuffs, the piclking method will "export" only
69
+ * a pointer on the shared mmap. This implies that if the GC pops out
70
+ * on the mmap, the unpickled instance will segfault...
71
+ * This implies that unpickled instances has limited capabilities (no
72
+ * way to get back the mmap.mmap instance).
73
+ * There is a way (using another shared mmap at module level ?) to
74
+ * implement a custom reference count on the mmap pointer in order
75
+ * to determine when to DECREF the mmap.mmap instance.
76
+ *
77
+ * @todo think about implementing a safe pickle/unpickle method
64 78
  * @ingroup pymod_pyrpn_RPNExprIter */
65 79
 typedef struct
66 80
 {
@@ -76,12 +90,17 @@ typedef struct
76 90
 	/**@brief Python tuple with instances of RPNExpr */
77 91
 	PyObject *expr;
78 92
 
79
-	/**@brief Python mmap.mmap instance representing rif memory map */
93
+	/**@brief Python mmap.mmap instance representing rif memory map 
94
+	 * @note NULL if unpickled instance */
80 95
 	PyObject *mmap;
81 96
 
82
-	/**@brief Memory map buffer allowing acces to underlying pointer */
97
+	/**@brief Memory map buffer allowing acces to underlying pointer 
98
+	 * @note unrelevant if unpickled instance */
83 99
 	Py_buffer mm_buff;
84 100
 
101
+	/**@brief Memory map buffer pointer */
102
+	void *_mmap;
103
+
85 104
 } PyRPNIterExpr_t;
86 105
 
87 106
 /**@brief RPNIterExpr.params static method
@@ -292,5 +311,7 @@ rpn_if_param_t *_rpnif_get_params(unsigned short pos_flag, unsigned short res_fl
292 311
  */
293 312
 PyObject *_rpnif_params_to_tuple(const rpn_if_param_t *params);
294 313
 
314
+int _rpnif_init_expr_tuple(PyRPNIterExpr_t *expr_self);
315
+
295 316
 #endif
296 317
 

+ 32
- 1
rpn_if.c Bestand weergeven

@@ -18,7 +18,8 @@
18 18
  */
19 19
 #include "rpn_if.h"
20 20
 
21
-rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap)
21
+rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap,
22
+		rpn_expr_t *init_exprs)
22 23
 {
23 24
 	rpn_if_t *res;
24 25
 	size_t i;
@@ -74,6 +75,36 @@ rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap)
74 75
 
75 76
 	res->rpn_args = &(res->rpn_res[params->rpn_sz]);
76 77
 
78
+	if(init_exprs)
79
+	{ // using existing exppressions, checking them
80
+		short err = 0;
81
+		for(size_t i=0; i<params->rpn_sz; i++)
82
+		{
83
+			if(init_exprs[i].args_count != params->rpn_argc)
84
+			{
85
+				err = 1;
86
+				snprintf(init_exprs[i].err_reason, 128,
87
+						"Expected %ld arguments but expression got %ld",
88
+						params->rpn_argc,
89
+						init_exprs[i].args_count);
90
+			}
91
+			else if(init_exprs[i].stack_sz != params->rpn_stack_sz)
92
+			{
93
+				err = 1;
94
+				snprintf(init_exprs[i].err_reason, 128,
95
+						"Expected %d element stack but expression got %d",
96
+						params->rpn_stack_sz,
97
+						init_exprs[i].stack_sz);
98
+			}
99
+		}
100
+		if(err)
101
+		{
102
+			goto rpn_malloc_err;
103
+		}
104
+		res->rpn = init_exprs;
105
+		return res;
106
+	}
107
+
77 108
 	res->rpn = malloc(sizeof(rpn_expr_t) * params->rpn_sz);
78 109
 	if(!res->rpn)
79 110
 	{

+ 4
- 1
rpn_if.h Bestand weergeven

@@ -118,9 +118,12 @@ struct rpn_if_s
118 118
 /**@brief Alloc a new @ref rpn_if_s using given parameters
119 119
  * @param params IF parameters
120 120
  * @param memmap A suitable memory map. If NULL given, a new mmap is used
121
+ * @param init_exprs If no NULL uses this expression (steal refs), else uses
122
+ * 	new empty expressions
121 123
  * @return A pointer on an allocated @ref rpn_if_s
122 124
  */
123
-rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap);
125
+rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap,
126
+		rpn_expr_t *init_exprs);
124 127
 
125 128
 /**@brief Deallocate an @ref rpn_if_s and its ressources and close associated 
126 129
  * @ref rpn_expr_s

+ 4
- 1
rpn_if_default.c Bestand weergeven

@@ -108,7 +108,10 @@ rpn_if_param_t* rpn_if_default_params(short pos_flag, short res_flag,
108 108
 		perror("Unable to alloc iterated function params");
109 109
 		return NULL;
110 110
 	}
111
-	res->data = data = (rpn_if_default_data_t*)(&(res[1]));
111
+	data = (rpn_if_default_data_t*)(res + 1);
112
+	bzero(res, sizeof(*res));
113
+	bzero(data, sizeof(*data));
114
+	res->data = data;
112 115
 
113 116
 	data->pos_flag = pos_flag;
114 117
 	data->res_flag = res_flag;

+ 1
- 1
rpn_if_default.h Bestand weergeven

@@ -87,7 +87,7 @@ struct rpn_if_default_data_s
87 87
 	 */
88 88
 	size_t *size_lim;
89 89
 
90
-	/** Number of dimention (if XDIM ndim = len(size_lim)-1) */
90
+	/** Number of dimensions (if XDIM ndim = len(size_lim)-1) */
91 91
 	size_t ndim;
92 92
 
93 93
 	/**@brief Store constant values to set mem given res_flag

+ 1
- 1
rpn_ifs.c Bestand weergeven

@@ -81,7 +81,7 @@ size_t rpn_ifs_add_if(rpn_ifs_t *rifs, unsigned int weight)
81 81
 
82 82
 	rifs->weight[rifs->if_sz] = weight;
83 83
 	//WRONG expr ARGUMENT !!!
84
-	rifs->rpn_if[rifs->if_sz] = rpn_if_new(&(rifs->params), rifs->mem);
84
+	rifs->rpn_if[rifs->if_sz] = rpn_if_new(&(rifs->params), rifs->mem, NULL);
85 85
 	if(!rifs->rpn_if[rifs->if_sz])
86 86
 	{
87 87
 		return 0;

+ 2
- 2
tests/test_rpn.c Bestand weergeven

@@ -255,7 +255,7 @@ int test_rpn_if_default()
255 255
 		return -1;
256 256
 	}
257 257
 	
258
-	rif = rpn_if_new(rif_param, NULL);
258
+	rif = rpn_if_new(rif_param, NULL, NULL);
259 259
 	if(!rif)
260 260
 	{
261 261
 		perror("rpn_if_new() failed");
@@ -282,7 +282,7 @@ int test_rpn_if_default2()
282 282
 		return -1;
283 283
 	}
284 284
 	
285
-	rif = rpn_if_new(rif_param, NULL);
285
+	rif = rpn_if_new(rif_param, NULL, NULL);
286 286
 	if(!rif)
287 287
 	{
288 288
 		perror("rpn_if_new() failed");

+ 1
- 1
tests/tests_rpn_mutate.py Bestand weergeven

@@ -61,7 +61,7 @@ class TestRpnExprMutation(unittest.TestCase):
61 61
     def test_random_int_mutation_params(self):
62 62
         """ Testing int random values for parameters checking resulting expr len """
63 63
         for _ in range(50):
64
-            m_params = [random.randint(0,20) for _ in range(4)]
64
+            m_params = [random.randint(1,20) for _ in range(4)]
65 65
             expt_len = (m_params[0]-m_params[1])/sum(m_params)
66 66
             expt_len = 0 if expt_len < 0 else expt_len
67 67
             params = [1] + m_params + [(0,0,0),(0,0,0)]

+ 30
- 1
tests/tests_rpniter.py Bestand weergeven

@@ -32,7 +32,7 @@ except (ImportError, NameError) as e:
32 32
           file=sys.stderr)
33 33
     raise e
34 34
 
35
-from utils import Progress
35
+from utils import *
36 36
 
37 37
 
38 38
 class TestRpnExprCopy(unittest.TestCase):
@@ -149,6 +149,35 @@ class TestRPNIterInit(unittest.TestCase):
149 149
                 rif = pyrpn.RPNIterExpr(*args)
150 150
                 self.assertEqual(rif.shape(), expt)
151 151
 
152
+    def test_pickling(self):
153
+        """ Test pickling/unpickling """
154
+        tests = [((pyrpn.const.POS_XY,
155
+                          pyrpn.const.RESULT_COUNT,
156
+                          (640,480)), (640,480)),
157
+                        ((pyrpn.const.POS_LINEAR,
158
+                          pyrpn.const.RESULT_BOOL,
159
+                          (1024,)), (1024,)),
160
+                        ((pyrpn.const.POS_LINEAR,
161
+                          pyrpn.const.RESULT_RGBA,
162
+                          (1024,)), (1024,4)),
163
+                        ((pyrpn.const.POS_XDIM,
164
+                          pyrpn.const.RESULT_BOOL,
165
+                          (2,640,480)),  (640,480)),
166
+                        ((pyrpn.const.POS_XDIM,
167
+                          pyrpn.const.RESULT_RGB,
168
+                          (5,13,37,13,12,42)), (13,37,13,12,42,3)),
169
+                ]
170
+        for args, expt in tests:
171
+            for _ in range(10):
172
+                rif = pyrpn.RPNIterExpr(*args)
173
+                for ex in rif.expressions:
174
+                    ex.mutate(n_mutations = 15)
175
+                with self.subTest(rif=rif):
176
+                    st = pickle.dumps(rif)
177
+                    rif2 = pickle.loads(st)
178
+                    st2 = pickle.dumps(rif2)
179
+                    self.assertEqual(st, st2)
180
+
152 181
     @skipIfNoNumpy()
153 182
     def test_buffer(self):
154 183
         """ Test the len on buffer interface using numpy """

+ 123
- 0
tests/utils.py Bestand weergeven

@@ -17,6 +17,8 @@
17 17
 #        along with geneifs.  if not, see <http://www.gnu.org/licenses/>.
18 18
 #
19 19
 
20
+from ctypes import *
21
+import struct
20 22
 import sys
21 23
 import math
22 24
 
@@ -78,4 +80,125 @@ class Progress:
78 80
             raise expt
79 81
 
80 82
 
83
+def RPNExpr_pickle_bin_dump(state):
84
+    """ Return a dict containing rpn state dump fields
85
+    """
86
+    ser_fields = ['token_sz', 'stack_sz', 'argc', 'state']
87
+    ser_expr = struct.unpack('QQQQ', state[:32])
88
+    ser_expr = dict(zip(ser_fields, ser_expr))
89
+    err = state[32:32+128]
90
+    if err == b'\x00' * 128:
91
+        err = None
92
+    ser_expr['err_reason'] = err
81 93
 
94
+    token_sz = 24
95
+    tok_start = 32+128
96
+    tok_end = tok_start + (token_sz * ser_expr['token_sz'])
97
+
98
+    tokens = [struct.unpack('QQQ', state[tok_start+(24*i):tok_start+(24*(i+1))])
99
+              for i in range(ser_expr['token_sz'])]
100
+
101
+    tok_types = ['op', 'arg', 'val']
102
+    #pretty_toks=[{'type': tok_types[tok[0]],
103
+    #              'type_raw': tok[0],
104
+    #              'v1': hex(tok[1]),
105
+    #              'v2': hex(tok[2])}
106
+    #             for tok in tokens]
107
+    pretty_toks = [(tok_types[tok[0]],)+tok for tok in tokens]
108
+    
109
+    ser_expr['tokens'] = pretty_toks
110
+    #ser_expr['tokens'] = tokens
111
+
112
+    stack_start = tok_end
113
+    stack_end = stack_start + (8*ser_expr['stack_sz'])
114
+    if len(state) < stack_end:
115
+        print("ERROR expt size = %d but got %d" % (stack_end, len(state)))
116
+        ser_expr['stack'] = 'FAULT'
117
+        return ser_expr
118
+    stack = struct.unpack('L'*ser_expr['stack_sz'], state[stack_start:stack_end])
119
+    ser_expr['stack'] = stack
120
+
121
+    return ser_expr
122
+
123
+
124
+
125
+def RPNIterExpr_pickle_bin_dump(dump):
126
+
127
+    import pprint
128
+
129
+    params_fields = ('mem_sz','value_sz','rpn_sz','rpn_argc','rpn_stack_sz',
130
+            'getarg_f','getres_f','data_ptr')
131
+    data_fields = ('pos_flag','res_flag', 'size_lim_ptr', 'ndim', 'const_val_ptr')
132
+
133
+    ptr = dump[0:8]
134
+    params = dump[8:72]
135
+    data = dump[72:104]
136
+
137
+    print("PARAMS : %r" % params)
138
+    print("DATA   : %r" % data)
139
+
140
+    params = struct.unpack('LLLLbPPP', params)
141
+    params = dict(zip(params_fields, params))
142
+
143
+    data = struct.unpack('hhPLP', data)
144
+    data = dict(zip(data_fields, data))
145
+
146
+    params['data'] = data
147
+
148
+    sz_sz = data['ndim'] + (1 if data['pos_flag'] == pyrpn.const.POS_XDIM else 0)
149
+    sz_end = 104 + 8*sz_sz
150
+    sz_lim = struct.unpack('L' * sz_sz, dump[104:sz_end])
151
+    data['sz_lim'] = sz_lim
152
+
153
+    const_sz = 0
154
+    if data['res_flag'] == pyrpn.const.RESULT_CONST:
155
+        const_sz = 1
156
+    elif data['res_flag'] == pyrpn.const.RESULT_CONST_RGBA:
157
+        const_sz = 4
158
+
159
+    const_end = sz_end + (8*const_sz)
160
+
161
+    if const_sz:
162
+        const_val = struct.unpack('L'*const_sz, dump[sz_end:const_end])
163
+    else:
164
+        const_val = []
165
+
166
+
167
+    print("SZLIM  : %r" % (dump[104:sz_end]))
168
+    print("CONST  : %r" % (dump[sz_end:const_end]))
169
+
170
+    data['const_val'] = const_val
171
+
172
+    rpn_sz_end = const_end + (8*params['rpn_sz'])
173
+
174
+
175
+    rpn_sz = struct.unpack('L'*params['rpn_sz'], dump[const_end:rpn_sz_end])
176
+    
177
+    ddump = {'params': params, 'rpn_sz': rpn_sz, 'expr': dict()}
178
+
179
+    rpn_buf_start = rpn_sz_end
180
+    for i in range(params['rpn_sz']):
181
+        rpn_buf_end = rpn_buf_start + rpn_sz[i]
182
+        rpn_state = dump[rpn_buf_start:rpn_buf_end]
183
+        print("RPN#%d[%d:%d] = %r" % (i, rpn_buf_start, rpn_buf_end, rpn_state))
184
+        try:
185
+            ddump['expr'][i] = RPNExpr_pickle_bin_dump(rpn_state)
186
+        except Exception as expt:
187
+            print("ERROR : ", expt)
188
+        rpn_buf_start = rpn_buf_end
189
+
190
+    return ddump
191
+
192
+
193
+def bindump(data):
194
+
195
+    for i in range(0,len(data), 0x10):
196
+        print("%02X " % i, data[i:i+0x10])
197
+    if len(data) % 0x10:
198
+        last = ((len(data) / 0x10)+1)
199
+        print("%02X " % last, data[last:])
200
+
201
+    
202
+
203
+
204
+    

Loading…
Annuleren
Opslaan