Implement RpnIterExpr.mutate method

- adds basic tests of the method
- bugfix 0 weights handling

.gitignore

@@ -0,0 +1,6 @@
+*.o
+*.so
+*.gcda
+*.gcno
+lcov.info
+lcov_html

Doxyfile.mk

@@ -46,13 +46,13 @@ INLINE_GROUPED_CLASSES = NO
 INLINE_SIMPLE_STRUCTS  = NO
 TYPEDEF_HIDES_STRUCT   = NO
 LOOKUP_CACHE_SIZE      = 0
-EXTRACT_ALL            = YES
-EXTRACT_PRIVATE        = NO
-EXTRACT_PACKAGE        = NO
-EXTRACT_STATIC         = NO
+EXTRACT_ALL            = NO
+EXTRACT_PRIVATE        = YES
+EXTRACT_PACKAGE        = YES
+EXTRACT_STATIC         = YES
 EXTRACT_LOCAL_CLASSES  = YES
-EXTRACT_LOCAL_METHODS  = NO
-EXTRACT_ANON_NSPACES   = NO
+EXTRACT_LOCAL_METHODS  = YES
+EXTRACT_ANON_NSPACES   = YES
 HIDE_UNDOC_MEMBERS     = NO
 HIDE_UNDOC_CLASSES     = NO
 HIDE_FRIEND_COMPOUNDS  = NO
@@ -83,7 +83,7 @@ QUIET                  = NO
 WARNINGS               = YES
 WARN_IF_UNDOCUMENTED   = YES
 WARN_IF_DOC_ERROR      = YES
-WARN_NO_PARAMDOC       = NO
+WARN_NO_PARAMDOC       = YES
 WARN_AS_ERROR          = NO
 WARN_FORMAT            = "$file:$line: $text"
 INPUT                  = ./
@@ -151,7 +151,6 @@ USE_HTAGS              = NO
 VERBATIM_HEADERS       = YES
 CLANG_ASSISTED_PARSING = NO
 ALPHABETICAL_INDEX     = YES
-COLS_IN_ALPHA_INDEX    = 5
 GENERATE_HTML          = YES
 HTML_OUTPUT            = html
 HTML_FILE_EXTENSION    = .html
@@ -199,14 +198,12 @@ PDF_HYPERLINKS         = YES
 USE_PDFLATEX           = YES
 LATEX_BATCHMODE        = NO
 LATEX_HIDE_INDICES     = NO
-LATEX_SOURCE_CODE      = NO
 LATEX_BIB_STYLE        = plain
 LATEX_TIMESTAMP        = NO
 GENERATE_RTF           = NO
 RTF_OUTPUT             = rtf
 COMPACT_RTF            = NO
 RTF_HYPERLINKS         = NO
-RTF_SOURCE_CODE        = NO
 GENERATE_MAN           = YES
 MAN_OUTPUT             = man
 MAN_EXTENSION          = .3
@@ -216,7 +213,6 @@ XML_OUTPUT             = xml
 XML_PROGRAMLISTING     = YES
 GENERATE_DOCBOOK       = NO
 DOCBOOK_OUTPUT         = docbook
-DOCBOOK_PROGRAMLISTING = NO
 GENERATE_AUTOGEN_DEF   = NO
 GENERATE_PERLMOD       = NO
 PERLMOD_LATEX          = NO
@@ -229,8 +225,6 @@ SKIP_FUNCTION_MACROS   = YES
 ALLEXTERNALS           = NO
 EXTERNAL_GROUPS        = YES
 EXTERNAL_PAGES         = YES
-PERL_PATH              = /usr/bin/perl
-CLASS_DIAGRAMS         = YES
 HIDE_UNDOC_RELATIONS   = YES
 HAVE_DOT               = YES
 DOT_NUM_THREADS        = 0

Makefile

@@ -3,13 +3,13 @@ NASM=nasm
 LD=ld
 
 ifeq ($(DEBUG), 1)
-	CFLAGS=-ggdb -fPIC -Wall -DDEBUG
+	CFLAGS=-ggdb -fPIC -Wall -DDEBUG -Wsign-compare
 	LDFLAGS=-g
 	NASMCFLAGS=-g -f elf64
 	#PYTHON=python3dm
 	PYTHON=python3-dbg
 else
-	CFLAGS=-fPIC -Wall -Werror
+	CFLAGS=-fPIC -Wall -Werror -DNDEBUG
 	LDFLAGS=-s
 	NASMCFLAGS=-f elf64
 	PYTHON=python3
@@ -19,43 +19,40 @@ PYTHON_CONFIG=$(PYTHON)-config
 PYTHON_CFLAGS=`$(PYTHON_CONFIG) --includes` `$(PYTHON_CONFIG) --cflags`
 PYTHON_LDFLAGS=-shared  -fPIC `$(PYTHON_CONFIG) --libs` `$(PYTHON_CONFIG) --ldflags|cut -d' ' -f1,2`
 
+C_SOURCES=$(wildcard *.c)
+C_OBJS=$(patsubst %.c,%.o,$(C_SOURCES))
+HEADERS=$(wildcard *.h)
+ASM_SOURCES=$(wildcard *.asm)
+ASM_OBJS=$(patsubst %.asm,%.o,$(ASM_SOURCES))
+OBJS=$(ASM_OBJS) $(C_OBJS)
+LIB=pyrpn.so
+
+CFLAGS_COV=$(CFLAGS) --coverage
+LDFLAGS_COV=$(LDFLAGS)
+C_OBJS_COV=$(patsubst %.c,%_cov.o,$(C_SOURCES))
+C_GCNO=$(patsubst %.o,%.gcno,$(C_OBJS_COV))
+C_GCDA=$(patsubst %.o,%.gcda,$(C_OBJS_COV))
+OBJS_COV=$(ASM_OBJS) $(C_OBJS_COV)
+LIB_COV=tests/pyrpn.so
+
 all: .deps pyrpn.so
 
-pyrpn.so: python_pyrpn.o python_rpnexpr.o python_if.o python_const.o rpn_lib.o rpn_jit.o rpn_parse.o rpn_mutation.o rpn_if.o rpn_if_default.o rpn_ifs.o
+$(C_OBJS): %.o: %.c $(HEADERS)
+	$(CC) $(PYTHON_CFLAGS) $(CFLAGS) -c -o $@ $<
+
+$(ASM_OBJS): %.o: %.asm $(HEADERS)
+	$(NASM) $(NASMCFLAGS) -o $@ $<
+
+$(LIB): $(OBJS)
 	$(LD) $(LDFLAGS) $(PYTHON_LDFLAGS) -o $@ $^
 
-python_pyrpn.o: python_pyrpn.c python_rpnexpr.h python_rpnexpr.o rpn_jit.o
-	$(CC) $(PYTHON_CFLAGS) $(CFLAGS) -c $<
 
-python_rpnexpr.o: python_rpnexpr.c python_rpnexpr.h rpn_jit.o
-	$(CC) $(PYTHON_CFLAGS) $(CFLAGS) -c $<
+$(C_OBJS_COV): %_cov.o: %.c $(HEADERS)
+	$(CC) $(PYTHON_CFLAGS) $(CFLAGS_COV) -c -o $@ $<
 
-python_if.o: python_if.c python_if.h
-	$(CC) $(PYTHON_CFLAGS) $(CFLAGS) -c $<
+$(LIB_COV): $(OBJS_COV)
+	gcc $(LDFLAGS_COV) $(PYTHON_LDFLAGS) -o $@ $^ --coverage -lgcov
 
-python_const.o: python_const.c python_const.h
-	$(CC) $(PYTHON_CFLAGS) $(CFLAGS) -c $<
-
-rpn_jit.o: rpn_jit.c rpn_jit.h rpn_parse.o rpn_lib.o
-	$(CC) $(CFLAGS) -c $<
-
-rpn_parse.o: rpn_parse.c rpn_parse.h rpn_lib.o
-	$(CC) $(CFLAGS) -c $<
-
-rpn_mutation.o: rpn_mutation.c rpn_mutation.h rpn_parse.o
-	$(CC) $(CFLAGS) -c $<
-
-rpn_if.o: rpn_if.c rpn_if.h rpn_jit.o
-	$(CC) $(CFLAGS) -c $<
-
-rpn_if_default.o: rpn_if_default.c rpn_if_default.h rpn_if.o
-	$(CC) $(CFLAGS) -c $<
-
-rpn_ifs.o: rpn_ifs.c rpn_ifs.h rpn_if.o
-	$(CC) $(CFLAGS) -c $<
-
-rpn_lib.o: rpn_lib.asm rpn_lib.h
-	$(NASM) $(NASMCFLAGS) -o $@ $<
 
 # Doxygen documentation
 doc: doc/.doxygen.stamp
@@ -75,20 +72,37 @@ doc/.doxygen.stamp: $(wildcard *.c) $(wildcard *.h) Doxyfile
 
 .PHONY: clean distclean checks runtest unittest benchmark
 
-checks: runtest unittest benchmark
+checks: runtest unittest
 
-benchmark: pyrpn.so
-	PYTHONPATH=`pwd` $(PYTHON) tests/benchmark.py 0x200 0x3000
+benchmark: $(LIB)
+	PYTHONPATH=`pwd` $(PYTHON) tests/benchmark.py -c 0x500 -i 0x2000 -s 0x50;  \
+	PYTHONPATH=`pwd` $(PYTHON) tests/benchmark.py -c 0x500 -i 0x2000 -s 0xa0;  \
+	PYTHONPATH=`pwd` $(PYTHON) tests/benchmark.py -c 0x500 -i 0x2000 -s 0x100;  \
+	PYTHONPATH=`pwd` $(PYTHON) tests/benchmark.py -c 0x500 -i 0x5000 -s 0x50; \
+	PYTHONPATH=`pwd` $(PYTHON) tests/benchmark.py -c 0x500 -i 0x5000 -s 0xa0; \
+	PYTHONPATH=`pwd` $(PYTHON) tests/benchmark.py -c 0x500 -i 0x5000 -s 0x100; \
+		   
+
+unittest: $(LIB_COV)
+	cd tests;\
+		$(PYTHON) -m unittest -v -f
+
+lcov.info: unittest runtest
+	lcov --no-external --base-directory ./ --capture --directory ./ --output-file $@
+
+lcov_html: lcov.info
+	genhtml $< --output-directory $@
+
+coverage: lcov_html
 
-unittest: pyrpn.so
-	PYTHONPATH=`pwd` $(PYTHON) -m unittest -v
 
 runtest:
 	make -C tests
 
 clean:
-	-rm -fv *.o pyrpn.so test;\
+	-rm -fv $(OBJS) $(LIB) test;\
 	rm -fRv doc/.doxygen.stamp doc/* Doxyfile;\
+	rm -fRv $(OBJS_COV) $(C_GCNO) $(C_GCDA) $(LIB_COV) lcov_html lcov.info;\
 	make -C tests clean
 
 distclean: clean

benchplot.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+gpdat=/tmp/bench.dat
+
+e=500
+
+min_i=10000
+max_i=100000
+it_i=1
+
+min_s=10
+max_s=10000
+it_s=30
+
+if [ $it_i -eq 1 ]
+then
+	inc_i=$max_i
+else
+	inc_i=$(expr $(expr $max_i - $min_i) / $(expr $it_i - 1))
+fi
+inc_s=$(expr $(expr $max_s - $min_s) / $(expr $it_s - 1))
+
+
+for i in $(seq $min_i $inc_i $max_i)
+do
+	for s in $(seq $min_s $inc_s $max_s)
+	do
+		echo "GO $e $i $s"
+		PYTHONPATH=./ python3 tests/benchmark.py -c $e -i $i -s $s -G $gpdat
+	done
+	echo "" >> $gpdat
+done

config.h

@@ -18,5 +18,24 @@
  */
 #ifndef __RPN_CONFIG__
 #define __RPN_CONFIG__
+
+/**@file config.h
+ * @brief Global definition for all C files
+ */
+
+/**@brief The programms is under GPL */
 #define _GNU_SOURCE
+
+/**@brief Allow fancy method declaration by indicating arguments list
+ * explicitly
+ * @param name char* The method name
+ * @param callback The C function to call
+ * @param flags int Python flags for method call options
+ * @param header char* List of arguments
+ * @param docstring char* The method documentation
+ */
+#define PYRPN_method(name, callback, flags, header, docstring) \
+{name, (PyCFunction)callback, flags, \
+	PyDoc_STR(name "("header ")\n--\n\n" docstring)}
+
 #endif

deploy.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+echo apt install htop vim git python3 python3-pip python3-numpy python3-sklearn screen nasm python3-tqdm python3-skimage
+echo pip3 install alphashape
+echo make

python_const.c

@@ -1,4 +1,11 @@
 #include "python_const.h"
+/**@file python_const.c
+ * @brief pyrpn.const code
+ * @ingroup python_ext
+ * @ingroup pymod_pyrpn_RPNIterParams
+ * @ingroup pymod_pyrpn_RPNTokenTypes
+ * @ingroup pymod_pyrpn_RPNMutationParams
+ */
 
 PyModuleDef rpnconstmodule = {
 	PyModuleDef_HEAD_INIT,
@@ -11,6 +18,117 @@ PyModuleDef rpnconstmodule = {
 	NULL
 };
 
+/* Various named tuple definition, instanciated in python_rpnexpr module init */
+/**@brief @ref pymod_pyrpn_RPNIterParams fields definition
+ * @ingroup pymod_pyrpn_RPNIterParams */
+static PyStructSequence_Field params_fields[] = {
+	{
+		.name = "argc",
+		.doc = "Number of arguments expected by expressions",
+	},
+	{	.name = "pos_flag",
+		.doc = "Position flag, indicating coordinate system",
+	},
+	{	.name =  "res_flag",
+		.doc = "Result flag, indicate how to handle expr results",
+	},
+	{	.name = "size_lim",
+		.doc = "Size limit, an array indicating the space size (depends \
+on pos_flag",
+	},
+	{
+		.name = "const_values",
+		.doc = "Constant values to use when setting value in space \
+(expr gives the position, and we set this value",
+	},
+	{
+		.name = "memory_size",
+		.doc = "The size of the underlying memory map data are \
+readed/written to",
+	},
+	{ NULL, NULL },
+};
+PyStructSequence_Desc rpnif_params_desc = {
+	.name = "RPNIterParams",
+	.doc = "Named tuple for RPNIter parameters",
+	.n_in_sequence = (sizeof(params_fields) / sizeof(*params_fields))-1,
+	.fields = params_fields,
+};
+PyTypeObject rpnif_params_SeqDesc;
+
+/**@brief @ref pymod_pyrpn_RPNTokenTypes field definition
+ * @ingroup pymod_pyrpn_RPNTokenTypes */
+static PyStructSequence_Field token_types_fields[] = {
+	{	.name = "op",
+		.doc = "Operation",
+	},
+	{	.name = "const",
+		.doc = "Constant value",
+	},
+	{	.name = "var",
+		.doc = "Variable (argument)",
+	},
+};
+PyStructSequence_Desc rpn_token_types_desc = {
+	.name = "RPNTokenTypes",
+	.doc = "Named tuple with token types as keys",
+	.n_in_sequence = 3,
+	.fields = token_types_fields,
+};
+PyTypeObject rpn_token_types_SeqDesc;
+
+/**@brief @ref pymod_pyrpn_RPNMutationParams field definition
+ * @ingroup pymod_pyrpn_RPNMutationParams */
+static PyStructSequence_Field mutation_params_fields[] = {
+	{	.name = "minimum_length",
+		.doc = "Expression minimum length",
+	},
+	{	.name =  "weight_add",
+		.doc = "The weight for 'add token' mutation",
+	},
+	{	.name = "weight_del",
+		.doc = "The weight for 'delete token' mutation",
+	},
+	{	.name = "weight_mut",
+		.doc = "The weight for 'mutate token' mutation",
+	},
+	{	.name = "weight_mut_soft",
+		.doc = "The weight for 'mutate soft (value not type) token' mutation",
+	},
+	{
+		.name = "weight_add_elt",
+		.doc = "The weights for each token types when adding a token",
+	},
+	{
+		.name = "weight_del_elt",
+		.doc= "The weights for each token types when deleting a token",
+	}
+};
+PyStructSequence_Desc rpn_mutation_params_desc = {
+	.name = "RPNMutationParams",
+	.doc = "Named tuple for mutation parameters",
+	.n_in_sequence = (sizeof(mutation_params_fields) / sizeof(*mutation_params_fields)),
+	.fields = mutation_params_fields,
+};
+PyTypeObject rpn_mutation_params_SeqDesc;
+
+
+rpn_mutation_params_t rpn_mutation_params_default = {
+	.min_len = 3,
+	.w_add = 1.25,
+	.w_del = 1.0,
+	.w_mut = 2.0,
+	.w_mut_soft = 4.0,
+	.w_add_elt = {1.0,1.0,1.0},
+	.w_mut_elt = {1.0,1.0,1.0},
+};
+
+/**@brief Adds a constant to @ref pymod_pyrpn_const module
+ * @param mod The @ref pymod_pyrpn_const module
+ * @param name The constant's name
+ * @param value The constant's value
+ * @return 0 or -1 on error
+ */
 int Py_rpnconst_add(PyObject* mod, const char* name, int value)
 {
 	PyObject *val;
@@ -24,12 +142,16 @@ int Py_rpnconst_add(PyObject* mod, const char* name, int value)
 	return 0;
 }
 
-PyObject *Py_rpnconst_init(void)
+/**@brief Module initialisation function
+ * @return The initialized module */
+PyObject *rpnconst_init(void)
 {
 	PyObject *mod;
 	mod = PyModule_Create(&rpnconstmodule);
 	if(mod == NULL) { return NULL; }
 
+	rpn_mutation_init_params(&rpn_mutation_params_default);
+
 	if(Py_rpnconst_add(mod, "POS_LINEAR", RPN_IF_POSITION_LINEAR) ||
 		Py_rpnconst_add(mod, "POS_XY", RPN_IF_POSITION_XY) ||
 		Py_rpnconst_add(mod, "POS_XDIM", RPN_IF_POSITION_XDIM) ||

python_const.h

@@ -27,14 +27,59 @@
 #include <Python.h>
 #include "structmember.h"
 
+#include "rpn_mutate.h"
 #include "rpn_if_default.h"
 
+/**@file python_const.h
+ * @brief Python pyrpn.const module headers
+ * @ingroup python_ext
+ * @ingroup pymod_pyrpn_const
+ */
+
+/**@defgroup pymod_pyrpn_const module pyrpn.const
+ * @brief A module containing constant values
+ *
+ * Constants values are @ref ifs_if_default_posflag
+ * and @ref ifs_if_default_resflag
+ * @ingroup pymod_pyrpn
+ */
+
 /**@brief pyrpn.const module specs
- * @ingroup python_module */
+ * @ingroup pymod_pyrpn */
 extern PyModuleDef rpnconstmodule;
 
+/**@defgroup pymod_pyrpn_RPNIterParams pyrpn.RPNIterParams
+ * @brief namedtuple representing @ref pymod_pyrpn_RPNExprIter parameters 
+ * @ingroup pymod_pyrpn */
+/**@brief RPNIterParams named tuple for RPNIterExpr parameters
+ * @ingroup pymod_pyrpn_RPNIterParams */
+extern PyTypeObject rpnif_params_SeqDesc;
+/**@brief @ref rpnif_params_SeqDesc named tuple description
+ * @ingroup pymod_pyrpn_RPNIterParams */
+extern PyStructSequence_Desc rpnif_params_desc;
+
+/**@defgroup pymod_pyrpn_RPNTokenTypes pyrpn.RPNTokenTypes
+ * @brief namedtuple with @ref pymod_pyrpn_token_Token subclass
+ * @ingroup pymod_pyrpn */
+/**@brief RPNTokenTypes named tuple with token types */
+extern PyTypeObject rpn_token_types_SeqDesc;
+/**@brief @ref rpn_token_types_SeqDesc named tuple description */
+extern PyStructSequence_Desc rpn_token_types_desc;
+
+/**@defgroup pymod_pyrpn_RPNMutationParams pyrpn.RPNMutationParams
+ * @brief namedtuple storing mutation parameters
+ * @ingroup pymod_pyrpn */
+/**@brief RPNMutationParams named tuple with mutation parameters */
+extern PyTypeObject rpn_mutation_params_SeqDesc;
+/**@brief @ref rpn_mutation_params_SeqDesc named tuple description */
+extern PyStructSequence_Desc rpn_mutation_params_desc;
+
+/**@brief Default values for mutations parameters */
+extern rpn_mutation_params_t rpn_mutation_params_default;
+
 /**@brief pyrpn.const module initialisation function
- * @ingroup python_module */
-PyObject *Py_rpnconst_init(void);
+ * @return The initialized module
+ * @ingroup pymod_pyrpn */
+PyObject *rpnconst_init(void);
 
 #endif

python_if.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020 Weber Yann
+ * Copyright (C) 2020,2023 Weber Yann
  * 
  * This file is part of pyrpn.
  * 
@@ -30,9 +30,18 @@
 #include "rpn_if.h"
 #include "rpn_if_default.h"
 #include "python_rpnexpr.h"
+#include "python_const.h"
 
-/**@defgroup python_if RPN Iterated Function Python class
- * @ingroup python_module
+/**@file python_if.h
+ * @brief Python RPNIterExpr type headers
+ * @ingroup python_ext
+ * @ingroup pymod_pyrpn_RPNExprIter
+ *
+ * This file is the header of the RPNIterExpr Python class
+ */
+
+/**@defgroup pymod_pyrpn_RPNExprIter class pyrpn.RPNExprIter
+ * @ingroup pymod_pyrpn
  * @brief Exposed Python class : RPNIterExpr
  *
  * Iterated expression are composed of RPN expressions, they are able to
@@ -42,39 +51,78 @@
  * @see ifs_if
  */
 
-/**@file python_if.h
- * @brief Python RPNIterExpr type headers
- * @ingroup python_if
- *
- * This file is the header of the RPNIterExpr Python class
- */
-
-/**@brief RPNIterExpr Python class methods list
- * @ingroup python_if */
-extern PyMethodDef RPNIterExpr_methods[];
-/**@brief RPNIterExpr Python class members list
- * @ingroup python_if */
-extern PyMemberDef RPNIterExpr_members[];
 /**@brief RPNIterExpr Python class type definition
- * @ingroup python_if */
+ * @ingroup pymod_pyrpn_RPNExprIter */
 extern PyTypeObject RPNIterExprType;
 
+/**@brief Points on Python's std mmap module */
+extern PyObject *mmap_module;
+/**@brief Python's mmap.mmap class */
+extern PyObject *mmap_cls;
+
 /**@brief Structure holding RPNIterExpr objects
- * @ingroup python_if */
+ *
+ * @warning Pickling methods are implemented for a limited usage.
+ * The mmap.mmap instance olded by the structure (sadly) cannot be
+ * pickled... In order to allow using RPNExprIter instances in
+ * multiprocessing stuffs, the piclking method will "export" only
+ * a pointer on the shared mmap. This implies that if the GC pops out
+ * on the mmap, the unpickled instance will segfault...
+ * This implies that unpickled instances has limited capabilities (no
+ * way to get back the mmap.mmap instance).
+ * There is a way (using another shared mmap at module level ?) to
+ * implement a custom reference count on the mmap pointer in order
+ * to determine when to DECREF the mmap.mmap instance.
+ *
+ * @todo think about implementing a safe pickle/unpickle method
+ * @ingroup pymod_pyrpn_RPNExprIter */
 typedef struct
 {
+	/** Python's type definition */
 	PyObject_VAR_HEAD;
 	
+	/** @brief Number of dimention in map */
+	size_t ndim;
+
 	/**@brief Pointer on @ref rpn_if_s */
 	rpn_if_t *rif;
 
+	/**@brief Python tuple with instances of RPNExpr */
+	PyObject *expr;
+
+	/**@brief Python mmap.mmap instance representing rif memory map 
+	 * @note NULL if unpickled instance */
+	PyObject *mmap;
+
+	/**@brief Memory map buffer allowing acces to underlying pointer 
+	 * @note unrelevant if unpickled instance */
+	Py_buffer mm_buff;
+
+	/**@brief Memory map buffer pointer */
+	void *_mmap;
+
+	/**@brief Pointer on params to free at del */
+	rpn_if_param_t *rif_params;
+
+	/**@brief If true, someone else (an ifs ?) will take care of freeing
+	 * the if at deletion */
+	short borrowed_if;
+
 } PyRPNIterExpr_t;
 
+/**@brief RPNIterExpr.params static method
+ * @param args Position arguments
+ * @param kwds Keyword arguments
+ * @return A new rpnif_params_SeqDesc instance
+ */
+PyObject* rpnif_params(PyObject *cls, PyObject *args, PyObject *kwds);
+
 /**@brief RpnIterExpr __new__ method
  * @param subtype Type of object being created (pyrpn.RPNIterExpr)
  * @param args positional arguments for subtype
- * @param kwargs keyword argumenrs for subtype
+ * @param kwds keyword argumenrs for subtype
  * @return The new Python RPNIterExpr object
+ * @ingroup pymod_pyrpn_RPNExprIter
  */
 PyObject* rpnif_new(PyTypeObject *subtype, PyObject* args, PyObject* kwds);
 
@@ -83,21 +131,104 @@ PyObject* rpnif_new(PyTypeObject *subtype, PyObject* args, PyObject* kwds);
  * @param args Positional arguments list
  * @param kwds Keywords arguments dict
  * @return 0 if no error else -1
- * @ingroup python_if
+ * @ingroup pymod_pyrpn_RPNExprIter
  */
 int rpnif_init(PyObject *self, PyObject *args, PyObject *kwds);
 
+/**@brief RPNIterExpr constructor with a borrowed expression
+ * @param borrowed_if Pointer on borrowed instance
+ * @param mmap Pointer on borrowed python mmap object (NULL if deserialized instance)
+ * @return 0 if no error else -1
+ */
+PyObject* rpnif_init_borrowed(rpn_if_t *borrowed_if, PyObject *mmap);
+
 /**@brief RPNIterExpr __del__ method 
  * @param self RPNExpr instance
- * @ingroup python_type
+ * @ingroup pymod_pyrpn_RPNExprIter
  */
 void rpnif_del(PyObject *self);
 
+/**@brief Buffer protocol request handler method
+ * @param self RPNIterExpr instance
+ * @param view A memoryview to fill depending on flags
+ * @param flags Indicates the request type
+ * @return -1 on error
+ * @note Used by RPNIterExpr and RPNIFS (proper way to implement it is base class)
+ * See python documentation at 
+ * https://docs.python.org/3/c-api/buffer.html#buffer-request-types
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+int rpnif_getbuffer(PyObject *self, Py_buffer *view, int flags);
+
+/**@brief Buffer protocol request release method
+ * @param self RPNIterExppr instance
+ * @param view The memoryview to release (free what we filled)
+ * See python documentation at 
+ * https://docs.python.org/3/c-api/buffer.html#buffer-request-types
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+void rpnif_releasebuffer(PyObject *self, Py_buffer *view);
+
+/**@brief Return a named tuple of custom rif data
+ * @param self RPNIterExpr instance
+ * @return A RPNIterParams named tuple
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+PyObject *rpnif_get_params(PyObject *self);
+
+/**@brief Set the mmap object used for data storage
+ * @param self RPNIterExpr instance
+ * @return None
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+PyObject *rpnif_set_mmap(PyObject *self, PyObject *mm_obj);
+
+/**@brief Return a tuple with data buffer's shape
+ * @param self RPNIterExpr instance
+ * @return A new ref on a tuple
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+PyObject *rpnif_shape(PyObject *self);
+
+/**@brief Runs an IF on given position
+ * @param self RPNInterExpr instance
+ * @param pos The start position
+ * @return A new position
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+PyObject *rpnif_step(PyObject *self, PyObject* pos);
+
+/**@brief Mutate the expressions in the IF
+ * @note For X mutations, an expression will be choosen randomly and mutate between
+ * 1 and X times, if mutations left to do we loop on expression choice.
+ * @param self
+ * @param args
+ * @param kwargs
+ * @return None
+ */
+PyObject* rpnif_mutate(PyObject *self, PyObject *args, PyObject  *kwargs);
+
+/**@brief Convert given arguments coordinates to position
+ * @param self RPNIterExpr instance
+ * @param argv Pointer on the array of arguments
+ * @param argc Argument count
+ * @return An integer
+ */
+PyObject *rpnif_to_pos(PyObject *self, PyObject** argv, Py_ssize_t argc);
+
+/**@brief Convert given position to coordinates tuple
+ * @param self RPNIterExpr instance
+ * @param pos Position, as integer, to convert
+ * @return A tuple
+ */
+PyObject *rpnif_from_pos(PyObject *self, PyObject* pos);
+
 /**@brief RPNIterExpr __getstate__ method for pickling
  * @param cls RPNIterExpr type object
  * @param noargs Not an argument...
  * @return A bytes Python instance suitable as argument for
  * @ref rpnexpr_setstate
+ * @ingroup pymod_pyrpn_RPNExprIter
  */
 PyObject* rpnif_getstate(PyObject *cls, PyObject *noargs);
 
@@ -107,20 +238,129 @@ PyObject* rpnif_getstate(PyObject *cls, PyObject *noargs);
  * rpnexpr_getstate
  * @return A bytes Python instance suitable as argument for
  * @ref rpnexpr_setstate
+ * @ingroup pymod_pyrpn_RPNExprIter
  */
 PyObject* rpnif_setstate(PyObject *cls, PyObject *state);
 
 /**@brief RPNIterExpr.__repr__()
  * @param self RPNIterExpr instance
- * @ingroup python_type
+ * @return A string representation of the instance
+ * @ingroup pymod_pyrpn_RPNExprIter
  */
 PyObject* rpnif_repr(PyObject *self);
 
 /**@brief RPNIterExpr.__str__()
  * @param self RPNIterExpr instance
- * @ingroup python_type
+ * @return A string representation of the instance
+ * @ingroup pymod_pyrpn_RPNExprIter
  */
 PyObject* rpnif_str(PyObject *self);
 
+/**@brief RPNExpr.__len__() method
+ * @param self RPNIterExpr instance
+ * @return A integer with the number of tokens in expression
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+Py_ssize_t rpnif_len(PyObject *self);
+
+/**@brief Sequence method for __getitem__ method
+ *
+ * Allow expressions access with integer indexes
+ * @param self RPNIterExpr instance
+ * @param idx Item index
+ * @return Corresponding RPNExpr or raise IndexError
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+PyObject* rpnif_expr_item(PyObject *self, Py_ssize_t idx);
+
+/**@brief Sequence method for __setitem__ method
+ *
+ * Allow to recompile an expressions using a str with integer indexes
+ * @param self RPNIterExpr instance
+ * @param idx Item index
+ * @param elt The code (str) to compile in indicated expression
+ * @return 0 if no error else -1
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+int rpnif_expr_ass_item(PyObject *self, Py_ssize_t idx, PyObject* elt);
+
+/**@brief Mapping method to access expressions by name
+ * @param self RPNIterExpr instance
+ * @param key An expression name (depending on parameters)
+ * @return The corresponding RPNExpr of raise KeyError
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+PyObject* rpnif_subscript(PyObject *self, PyObject *key);
+
+/**@brief Mapping method to recompile an expression by name
+ * @param self RPNIterExpr instance
+ * @param key An expression name (depending on parameters)
+ * @param elt The code (str) to compile in indicated expression
+ * @return 0 if no error else -1
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+int rpnif_ass_subscript(PyObject *self, PyObject *key, PyObject *elt);
+
+/**@brief Mimics dict.keys() method
+ * @param self RPNIterExpr instance
+ * @return A tuple with expressions name
+ * @ingroup pymod_pyrpn_RPNExprIter */
+PyObject *rpnif_keys(PyObject *self);
+
+/**@brief Mimics dict.values() method
+ * @param self RPNIterExpr instance
+ * @return A tuple with RPNExpr instances
+ * @ingroup pymod_pyrpn_RPNExprIter */
+PyObject *rpnif_values(PyObject *self);
+
+/**@brief Mimics dict.items() method
+ * @param self RPNIterExpr instance
+ * @return A tuple with tuples(key, value)
+ * @ingroup pymod_pyrpn_RPNExprIter */
+PyObject *rpnif_items(PyObject *self);
+
+/**@brief Returns rpn_if_params given arguments
+ * @param pos_flag a position flag
+ * @param res_flag a result flag
+ * @param size_lim a tuple with size limits
+ * @param const_values a tuple wiith constant values
+ * @param stack_size a stack size
+ */
+rpn_if_param_t *_rpnif_get_params(unsigned short pos_flag, unsigned short res_flag,
+		PyObject *size_lim, PyObject *const_values,
+		unsigned short stack_size);
+
+/**@brief Converts rpn_if_params to named tuple
+ * @param params The rpnif params
+ * @return A named tuple
+ */
+PyObject *_rpnif_params_to_tuple(const rpn_if_param_t *params);
+
+
+/**@brief Implementation of both RPNIterExpr.to_pos and RPNIFS.to_pos */
+PyObject *_rpnif_to_pos(rpn_if_default_data_t *rif_data, PyObject** argv, Py_ssize_t argc);
+
+/**@brief Implementation of bot RPNIterExpr.to_pos and RPNIFS.from_pos */
+PyObject *_rpnif_from_pos(rpn_if_default_data_t *rif_data, PyObject* _pos);
+
+
+/**@brief Buffer protocol request handler method when no python mmap exist
+ *        (deserialized instance)
+ * @param self RPNIterExpr instance
+ * @param view A memoryview to fill depending on flags
+ * @param flags Indicates the request type
+ * @return -1 on error
+ * @note Used by RPNIterExpr and RPNIFS (proper way to implement it is base class)
+ * See python documentation at 
+ * https://docs.python.org/3/c-api/buffer.html#buffer-request-types
+ * @ingroup pymod_pyrpn_RPNExprIter
+ */
+int _rpnif_getbuffer_nopymap(PyObject *self, const rpn_if_param_t *params,
+		void *buf, Py_buffer *view, int flags);
+void _rpnif_releasebuffer_nopymap(PyObject *self, Py_buffer *view);
+
+/**@todo doc */
+int _rpnif_init_expr_tuple(PyRPNIterExpr_t *expr_self);
+
 #endif
 

python_ifs.c

@@ -0,0 +1,487 @@
+/*
+ * Copyright (C) 2020,2023 Weber Yann
+ * 
+ * This file is part of pyrpn.
+ * 
+ * pyrpn is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * 
+ * pyrpn is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with pyrpn.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "python_ifs.h"
+
+/**@file python_ifs.c */
+
+static PyMethodDef RPNIFS_methods[] = {
+	PYRPN_method("weights", rpnifs_weights, 
+			METH_FASTCALL,
+			"self, weights=None",
+			"Get or set the weights list.\n\n\
+When setting the weight list, the weight list len sets the number of functions \
+in the system."),
+	PYRPN_method("position", rpnifs_position, 
+			METH_FASTCALL,
+			"self, position=None",
+			"Get or set the current position"),
+	PYRPN_method("run", rpnifs_run,
+			METH_FASTCALL,
+			"self, steps, rand_bytes=None, /",
+			"Run ifs steps times"),
+	PYRPN_method("to_pos", rpnifs_to_pos,
+			METH_FASTCALL,
+			"self, *args, /",
+			"Return a position (int) from a coordinates given as"
+			"argument."),
+	PYRPN_method("from_pos", rpnifs_from_pos,
+			METH_O,
+			"self, position, /",
+			"Return a coordinates tuple from given position."),
+	{NULL} // Sentinel
+};
+
+static PyMemberDef RPNIFS_members[] = {
+	{"expressions", T_OBJECT, offsetof(PyRPNIFS_t, rpn_if), READONLY,
+		"The tuple with RPNIterExpr instances"},
+	{"mmap", T_OBJECT, offsetof(PyRPNIFS_t, mmap), READONLY,
+		"The mmap storing data"},
+	{NULL} // sentinel
+};
+
+static PySequenceMethods RPNIFS_seq_methods = {
+	.sq_length = rpnifs_len,
+	.sq_item = rpnifs_expr_item,
+	.sq_ass_item = rpnifs_expr_ass_item,
+};
+
+static PyBufferProcs RPNIFS_as_buffer = {
+	(getbufferproc)rpnifs_getbuffer,
+	(releasebufferproc)rpnifs_releasebuffer,
+};
+
+
+PyTypeObject RPNIFSType = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name = "pyrpn.RPNIFS",
+	.tp_basicsize = sizeof(PyRPNIFS_t),
+	.tp_itemsize = 0,
+	.tp_del = rpnifs_del,
+	.tp_repr = rpnifs_repr,
+	.tp_as_sequence = &RPNIFS_seq_methods,
+	.tp_str = rpnifs_str,
+	.tp_as_buffer = &RPNIFS_as_buffer,
+	.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+	.tp_doc = "RPN IFS",
+	.tp_methods = RPNIFS_methods,
+	.tp_members = RPNIFS_members,
+	.tp_init = rpnifs_init,
+	.tp_new = rpnifs_new,
+};
+
+PyObject *rpnifs_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)
+{
+	PyObject *ret, *err;
+	PyRPNIFS_t *ifs_self;
+	ret = PyType_GenericNew(subtype, args, kwds);
+	if((err = PyErr_Occurred()))
+	{
+		Py_DECREF(err);
+		return ret;
+	}
+	ifs_self = (PyRPNIFS_t*)ret;
+	ifs_self->rpn_if = NULL;
+	ifs_self->rpn_expr = NULL;
+	ifs_self->rifs = NULL;
+	ifs_self->mmap = NULL;
+
+	return ret;
+}
+
+
+int rpnifs_init(PyObject *self, PyObject *args, PyObject *kwds)
+{
+	PyRPNIFS_t *ifs_self;
+	char *names[] = {"pos_flag", "res_flag", "size_lim",
+		"const_values", "stack_size", "mmap", NULL};
+	unsigned short pos_flag, res_flag, stack_size;
+	PyObject *lim_obj, *const_values_obj, *mmap_obj;
+
+	ifs_self = (PyRPNIFS_t*)self;
+
+	// default values
+	stack_size = 16;
+	const_values_obj = lim_obj = mmap_obj = NULL;
+	ifs_self->rifs = NULL;
+
+	if(!PyArg_ParseTupleAndKeywords(args, kwds, "HHO|OHO:RPNIFS.__init__",
+		names,
+		&pos_flag, &res_flag, &lim_obj, &const_values_obj,
+		&stack_size, &mmap_obj))
+	{
+		return -1;
+	}
+	
+	rpn_if_param_t *rif_params = _rpnif_get_params(pos_flag, res_flag,
+			lim_obj, const_values_obj, stack_size);
+	if(!rif_params)
+	{
+		return -1;
+	}
+
+	const Py_ssize_t expt_sz = rif_params->mem_sz * rif_params->value_sz;
+	if(!mmap_obj)
+	{
+		PyObject *fileno = PyLong_FromLong(-1);
+		PyObject *length = PyLong_FromSize_t(expt_sz);
+		mmap_obj = PyObject_CallFunctionObjArgs(mmap_cls, fileno, length,
+				NULL);
+		Py_DECREF(fileno);
+		Py_DECREF(length);
+		if(PyErr_Occurred())
+		{
+			return -1;
+		}
+	}
+	else
+	{
+		if(!PyObject_TypeCheck(mmap_obj, (PyTypeObject*)mmap_cls))
+		{
+			PyErr_SetString(PyExc_TypeError,
+					"The mmap argument MUST be an instance \
+of mmap.mmap");
+			return -1;
+		}
+		/**@todo check if mmap is shared & writable ? */
+		if(PyObject_Length(mmap_obj) != (Py_ssize_t)expt_sz)
+		{
+			PyErr_Format(PyExc_ValueError,
+				"Expected mmap lenght is %ld but mmap with \
+length %ld provided",
+				rif_params->mem_sz, PyObject_Length(mmap_obj));
+			return -1;
+		}
+	}
+
+	if(PyObject_GetBuffer(mmap_obj, &ifs_self->mm_buff,
+			PyBUF_CONTIG) == -1)
+	{
+		return -1;
+	}
+
+	ifs_self->mmap = mmap_obj;
+	ifs_self->_mmap = ifs_self->mm_buff.buf;
+
+	ifs_self->ifs = rpn_ifs_new(rif_params, ifs_self->mm_buff.buf);
+	if(!ifs_self->ifs)
+	{
+		PyErr_Format(PyExc_RuntimeError,
+			"Error initializing ifs: %s", strerror(errno));
+		return -1;
+	}
+	
+	ifs_self->rpn_if = NULL;
+	if(_rpnifs_update_if_tuple(ifs_self) < 0)
+	{
+		return -1;
+	}
+	return 0;
+}
+
+
+void rpnifs_del(PyObject *self)
+{
+	PyRPNIFS_t *ifs_self = (PyRPNIFS_t*)self;
+
+	rpn_ifs_free(ifs_self->ifs);
+	free(ifs_self->ifs);
+	if(ifs_self->mmap)
+	{
+		if(ifs_self->mm_buff.buf)
+		{
+			PyBuffer_Release(&ifs_self->mm_buff);
+			ifs_self->mm_buff.buf = NULL;	
+		}
+		Py_DECREF(ifs_self->mmap);
+		ifs_self->mmap = NULL;
+	}
+}
+
+
+PyObject *rpnifs_to_pos(PyObject *self, PyObject** argv, Py_ssize_t argc)
+{
+	PyRPNIFS_t *ifs_self = (PyRPNIFS_t*)self;
+	rpn_if_default_data_t *rif_data = (rpn_if_default_data_t*)ifs_self->ifs->params.data;
+	return _rpnif_to_pos(rif_data, argv, argc);
+}
+
+
+PyObject *rpnifs_from_pos(PyObject *self, PyObject* _pos)
+{
+	PyRPNIFS_t *ifs_self = (PyRPNIFS_t*)self;
+	rpn_if_default_data_t *rif_data = (rpn_if_default_data_t*)ifs_self->ifs->params.data;
+	return _rpnif_from_pos(rif_data, _pos);
+}
+
+
+
+PyObject *rpnifs_weights(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
+{
+	PyRPNIFS_t *ifs_self = (PyRPNIFS_t*)self;
+	PyObject *iter;
+	unsigned int *weights;
+
+	if(nargs == 1 && args[0] != Py_None)
+	{
+		// Try to update weights
+		const Py_ssize_t len = PyObject_Length(args[0]);
+		if(PyErr_Occurred())
+		{
+			return NULL;
+		}
+		/*
+		if((size_t)len != ifs_self->ifs->if_sz)
+		{
+			PyErr_Format(PyExc_ValueError,
+				"Expected %ld weights but argument length is %ld",
+				ifs_self->ifs->if_sz, len);
+			return NULL;
+		}
+		*/
+		weights = malloc(sizeof(unsigned int)*len);
+		if(len && !weights)
+		{
+			PyErr_Format(PyExc_MemoryError,
+				"Unable to allocate weights : %s",
+				strerror(errno));
+			return NULL;
+		}
+		iter = PyObject_GetIter(args[0]);
+		if(PyErr_Occurred())
+		{
+			goto err_weights;
+		}
+		for(size_t i=0; i<(unsigned int)len; i++)
+		{
+			PyObject *item = PyIter_Next(iter);
+			if(PyErr_Occurred())
+			{
+				goto err_weights;
+			}
+			const unsigned long w = PyLong_AsUnsignedLong(item);
+			Py_DECREF(item);
+			if(PyErr_Occurred())
+			{
+				goto err_weights;
+			}
+			weights[i] = w;
+		}
+		Py_DECREF(iter);
+		rpn_ifs_set_if_count(ifs_self->ifs, len, weights);
+		free(weights);
+	}
+
+	// return weights in a new tuple
+	PyObject *ret = PyTuple_New(ifs_self->ifs->if_sz);
+	if(PyErr_Occurred())
+	{
+		return NULL;
+	}
+	for(size_t i=0; i<ifs_self->ifs->if_sz; i++)
+	{
+		const unsigned int weight = ifs_self->ifs->weight[i];
+		PyObject *item = PyLong_FromUnsignedLong(weight);
+		if(PyErr_Occurred()) { return NULL; }
+		PyTuple_SET_ITEM(ret, i, item);
+	}
+
+	if(_rpnifs_update_if_tuple(ifs_self) < 0)
+	{
+		return NULL;
+	}
+	return ret;
+
+err_weights:
+	free(weights);
+	Py_DECREF(iter);
+	return NULL;
+}
+
+
+PyObject *rpnifs_position(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
+{
+	PyRPNIFS_t *ifs_self = (PyRPNIFS_t*)self;
+
+	if(nargs == 1 && args[0] != Py_None)
+	{
+		size_t pos = PyLong_AsSize_t(args[0]);
+		if(PyErr_Occurred()) { return NULL; }
+		if(pos > ifs_self->ifs->params.mem_sz)
+		{
+			pos %= ifs_self->ifs->params.mem_sz;
+#ifndef NDEBUG
+			PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
+					"Given position %R overflows and stored as %lu",
+					args[0], pos);
+#endif
+		}
+		ifs_self->ifs->pos = pos;
+	}
+
+	return PyLong_FromSize_t(ifs_self->ifs->pos);
+}
+
+PyObject *rpnifs_run(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
+{
+	PyRPNIFS_t *ifs_self = (PyRPNIFS_t*)self;
+	if(nargs == 0 || nargs > 2)
+	{
+		PyErr_Format(PyExc_ValueError,
+				"Expected 1 or 2 arguments but %ld given",
+				nargs);
+		return NULL;
+	}
+
+	size_t steps = PyLong_AsSize_t(args[0]);
+	if(PyErr_Occurred())
+	{
+		return NULL;
+	}
+	else if(steps == 0)
+	{
+		PyErr_Format(PyExc_ValueError,
+				"Steps arguments must be > 0");
+		return NULL;
+	}
+
+	unsigned char *rnd_buf = NULL;
+	if(nargs == 2)
+	{
+		// Python bytes given as random source
+		const Py_ssize_t sz = PyBytes_Size(args[1]);
+		if(PyErr_Occurred()) { goto err; }
+		if((size_t)sz != sizeof(char)*steps)
+		{
+			PyErr_Format(PyExc_ValueError,
+					"Given random bytes are too small."
+					"Expected %ld bytes but got %ld",
+					sizeof(char)*steps,
+					sz);
+			goto err;
+		}
+		rnd_buf = (unsigned char*)PyBytes_AsString(args[1]);
+	}
+
+	const int ret = rpn_ifs_run(ifs_self->ifs, steps, rnd_buf);
+	if(ret < 0)
+	{
+		PyErr_Format(PyExc_RuntimeError,
+				"Unable to run IFS : %s",
+				strerror(errno));
+		return NULL;
+	}
+	
+	Py_RETURN_NONE;
+
+err:
+	return NULL;
+}
+
+PyObject *rpnifs_str(PyObject *self)
+{
+	PyErr_SetString(PyExc_NotImplementedError, "TODO");
+	return NULL;
+}
+
+
+PyObject *rpnifs_repr(PyObject *self)
+{
+	PyErr_SetString(PyExc_NotImplementedError, "TODO");
+	return NULL;
+}
+
+
+Py_ssize_t rpnifs_len(PyObject *self)
+{
+	PyRPNIFS_t *ifs_self = (PyRPNIFS_t*)self;
+	return (Py_ssize_t)ifs_self->ifs->if_sz;
+}
+
+
+PyObject *rpnifs_expr_item(PyObject *self, Py_ssize_t idx)
+{
+	PyRPNIFS_t *ifs_self = (PyRPNIFS_t*)self;
+	Py_ssize_t _idx = idx;
+
+	if(idx < 0)
+	{
+		_idx = ifs_self->ifs->if_sz - 1 + idx;
+	}
+	if(_idx < 0 || (size_t) _idx >= ifs_self->ifs->params.rpn_sz)
+	{
+		PyErr_Format(PyExc_IndexError,
+			"There is %ld expressions in the system but index %ld asked",
+			ifs_self->ifs->params.rpn_sz, idx);
+		return NULL;
+	}
+	PyObject *ret = PyTuple_GET_ITEM(ifs_self->rpn_if, _idx);
+	Py_INCREF(ret);
+	return ret;
+}
+
+
+int rpnifs_expr_ass_item(PyObject *self, Py_ssize_t idx, PyObject *value)
+{
+	PyErr_SetString(PyExc_NotImplementedError, "TODO");
+	return -1;
+}
+
+
+int rpnifs_getbuffer(PyObject *self, Py_buffer *view, int flags)
+{
+	PyRPNIFS_t *ifs_self = (PyRPNIFS_t*)self;
+
+	if(ifs_self->mmap)
+	{
+		return PyObject_GetBuffer(ifs_self->mmap, view, flags);
+	}
+	return _rpnif_getbuffer_nopymap(self, &ifs_self->ifs->params,
+			ifs_self->_mmap, view, flags);
+}
+
+void rpnifs_releasebuffer(PyObject *self, Py_buffer *view)
+{
+	PyRPNIFS_t *ifs_self = (PyRPNIFS_t*)self;
+	if(ifs_self->mmap)
+	{
+		return PyBuffer_Release(view);
+	}
+	return _rpnif_releasebuffer_nopymap(self, view);
+}
+
+int _rpnifs_update_if_tuple(PyRPNIFS_t *ifs_self)
+{
+	if(ifs_self->rpn_if)
+	{
+		Py_DECREF(ifs_self->rpn_if);
+	}
+	ifs_self->rpn_if = PyTuple_New(ifs_self->ifs->if_sz);
+	for(size_t i=0; i<ifs_self->ifs->if_sz; i++)
+	{
+		PyObject *py_if = rpnif_init_borrowed(ifs_self->ifs->rpn_if[i],
+				ifs_self->mmap);
+		if((!py_if) || PyErr_Occurred())
+		{
+			return -1;
+		}
+		PyTuple_SET_ITEM(ifs_self->rpn_if, i, py_if);
+	}
+	return 0;
+}
+

python_ifs.h

@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2020,2023 Weber Yann
+ * 
+ * This file is part of pyrpn.
+ * 
+ * pyrpn is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * 
+ * pyrpn is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with pyrpn.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _PYTHON_IFS_H__
+#define _PYTHON_IFS_H__
+
+#include "config.h"
+
+#include <errno.h>
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include "structmember.h"
+
+#include "rpn_if.h"
+#include "rpn_if_default.h"
+#include "rpn_ifs.h"
+#include "python_rpnexpr.h"
+#include "python_if.h"
+#include "python_const.h"
+
+
+/**@file python_ifs.h */
+
+/** @brief RPNIFS Python class type definition */
+extern PyTypeObject RPNIFSType;
+
+/**@brief Points on Python's std mmap module */
+extern PyObject *mmap_module;
+/**@brief Python's mmap.mmap class */
+extern PyObject *mmap_cls;
+
+/** @brief Structure holding RPNIFS attributes */
+typedef struct
+{
+	PyObject_VAR_HEAD;
+
+	/** @brief Tuple with RPNIterExpr instances */
+	PyObject *rpn_if;
+
+	/** @brief Tuple with RPNExpr instances */
+	PyObject *rpn_expr;
+
+	/** @brief IFS pointer */
+	rpn_ifs_t *ifs;
+
+	/** @brief Pointer on IF expressions in the system */
+	rpn_if_t **rifs;
+
+
+	/**@brief Python mmap.mmap instance representing rif memory map 
+	 * @note NULL if unpickled instance */
+	PyObject *mmap;
+
+	/**@brief Memory map buffer allowing acces to underlying pointer 
+	 * @note unrelevant if unpickled instance */
+	Py_buffer mm_buff;
+
+	/**@brief Memory map buffer pointer */
+	void *_mmap;
+
+} PyRPNIFS_t;
+
+
+PyObject *rpnifs_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds);
+int rpnifs_init(PyObject *self, PyObject *args, PyObject *kwds);
+void rpnifs_del(PyObject *self);
+
+
+PyObject *rpnifs_to_pos(PyObject *self, PyObject** argv, Py_ssize_t argc);
+PyObject *rpnifs_from_pos(PyObject *self, PyObject* _pos);
+
+PyObject *rpnifs_weights(PyObject *self, PyObject *const *args, Py_ssize_t nargs);
+PyObject *rpnifs_position(PyObject *self, PyObject *const *args, Py_ssize_t nargs);
+PyObject *rpnifs_run(PyObject *self, PyObject *const *args, Py_ssize_t nargs);
+
+PyObject *rpnifs_str(PyObject *self);
+PyObject *rpnifs_repr(PyObject *self);
+
+Py_ssize_t rpnifs_len(PyObject *self);
+PyObject *rpnifs_expr_item(PyObject *self, Py_ssize_t idx);
+int rpnifs_expr_ass_item(PyObject *self, Py_ssize_t idx, PyObject *value);
+
+int rpnifs_getbuffer(PyObject *self, Py_buffer *view, int flags);
+void rpnifs_releasebuffer(PyObject *self, Py_buffer *view);
+
+int _rpnifs_update_if_tuple(PyRPNIFS_t *ifs_self);
+
+
+#endif

python_pyrpn.c

@@ -21,15 +21,21 @@
 /**@file python_pyrpn.c
  * @brief Python module & type definition
  * @ingroup python_ext
+ * @ingroup python_pyrpn
  *
  * This file contains pyrpn Python module definition
  */
 
-PyMethodDef rpnmodule_methods[] = {
-	{"get_ops", (PyCFunction)pyrpn_ops, METH_NOARGS,
-		"Returns a valid operands dict"},
-	{"random_expr", (PyCFunction)pyrpn_random, METH_VARARGS | METH_KEYWORDS,
-		"Return a random RPN expression"},
+/**@brief pyrpn module's method definition */
+static PyMethodDef rpnmodule_methods[] = {
+	PYRPN_method("get_ops", pyrpn_ops,
+			METH_NOARGS,
+			"/",
+			"Return a dict with valid operands"),
+	PYRPN_method("random_expr", pyrpn_random,
+			METH_VARARGS | METH_KEYWORDS,
+			"args_count, token_count",
+			"Return a random RPN expression string"),
 	{NULL} // Sentinel
 };
 
@@ -45,62 +51,155 @@ PyModuleDef rpnmodule = {
 	NULL	// m_free
 };
 
+PyObject *mmap_module = NULL;
+PyObject *mmap_cls = NULL;
+
 PyMODINIT_FUNC
 PyInit_pyrpn(void)
 {
+	if(mmap_module == NULL)
+	{
+		mmap_module = PyImport_ImportModule("mmap");
+		if(PyErr_Occurred()) { return NULL; }
+		else if(!mmap_module)
+		{
+			PyErr_Format(PyExc_ImportError, "Unable to import mmap module");
+			return NULL;
+		}
+	}
+	if(mmap_cls == NULL)
+	{
+		mmap_cls = PyObject_GetAttrString(mmap_module, "mmap");
+		if(PyErr_Occurred()) { return NULL; }
+		else if (!mmap_cls)
+		{
+			PyErr_Format(PyExc_ImportError, "Unable to import mmap.mmap");
+			return NULL;
+		}
+	}
+	
 
-	PyObject *mod, *const_mod;
+	PyObject *mod, *const_mod, *tokens_mod;
 	// init module & globals
 	mod = PyModule_Create(&rpnmodule);
 	if(mod == NULL) { return NULL; }
 	
 	//init constants module
-	const_mod = Py_rpnconst_init();
+	const_mod = rpnconst_init();
 	if(const_mod == NULL)
 	{
-		Py_DECREF(mod);
-		return NULL;
+		goto fail_init;
 	}
 	Py_INCREF(const_mod);
 	if(PyModule_AddObject(mod, "const", const_mod) < 0)
 	{
-		Py_DECREF(const_mod);
-		Py_DECREF(mod);
-		return NULL;
+		goto fail_add_const;
+	}
+
+	//init tokens module
+	tokens_mod = rpntokens_module_init();
+	if(tokens_mod == NULL)
+	{
+		goto fail_add_const;
+	}
+	Py_INCREF(tokens_mod);
+	if(PyModule_AddObject(mod, "tokens", tokens_mod) < 0)
+	{
+		goto fail_add_tokens;
 	}
 
 	// Init RPNExpr type
 	if(PyType_Ready(&RPNExprType) < 0)
 	{
-		return NULL;
+		goto fail_expr_type_ready;
 	}
 
 	// Add type to module
 	Py_INCREF(&RPNExprType);
 	if(PyModule_AddObject(mod, "RPNExpr", (PyObject*)&RPNExprType) < 0)
 	{
-		Py_DECREF(&RPNExprType);
-		Py_DECREF(mod);
-		Py_DECREF(const_mod);
-		return NULL;
+		goto fail_add_rpnexpr;
 	}
 
 	// Init RPNIterExpr type
 	if(PyType_Ready(&RPNIterExprType) < 0)
 	{
-		return NULL;
+		goto fail_iter_type_ready;
 	}
 
 	Py_INCREF(&RPNIterExprType);
 	if(PyModule_AddObject(mod, "RPNIterExpr", (PyObject*)&RPNIterExprType) < 0)
 	{
-		Py_DECREF(&RPNExprType);
-		Py_DECREF(&RPNIterExprType);
-		Py_DECREF(mod);
-		return NULL;
+		goto fail_add_iter;
+	}
+
+	if(PyType_Ready(&RPNIFSType) < 0)
+	{
+		goto fail_ifs_type_ready;
+	}
+
+	Py_INCREF(&RPNIFSType);
+	if(PyModule_AddObject(mod, "RPNIFS", (PyObject*)&RPNIFSType) < 0)
+	{
+		goto fail_add_ifs;
+	}
+
+	// Named tuple for RPNIterExpr's params
+	PyStructSequence_InitType(&rpnif_params_SeqDesc,&rpnif_params_desc);
+	Py_INCREF(&rpnif_params_SeqDesc);
+	/*
+	PyObject_SetAttrString((PyObject*)rpnif_params_SeqDesc, "__module__",
+		mod);
+	*/
+	if(PyModule_AddObject(mod, "RPNIterExprParams",
+		(PyObject*)&rpnif_params_SeqDesc) < 0)
+	{
+		goto fail_add_iter_expr_params;
+	}
+
+	// Named tuple for token types
+	PyStructSequence_InitType(&rpn_token_types_SeqDesc,
+			&rpn_token_types_desc);
+	Py_INCREF(&rpn_token_types_SeqDesc);
+	if(PyModule_AddObject(mod, "RPNTokenTypesTuple",
+				(PyObject*)&rpn_token_types_SeqDesc) < 0)
+	{
+		goto fail_add_token_types_tuple;
+	}
+
+	// Named tuple for mutation parameters
+	PyStructSequence_InitType(&rpn_mutation_params_SeqDesc,
+			&rpn_mutation_params_desc);
+	Py_INCREF(&rpn_mutation_params_SeqDesc);
+	if(PyModule_AddObject(mod, "RPNMutationParamsTuple",
+				(PyObject*)&rpn_mutation_params_SeqDesc) < 0)
+	{
+		goto fail_add_mutation_params_tuple;
 	}
 
 	return mod;
+
+fail_add_mutation_params_tuple:
+	Py_DECREF(&rpn_token_types_SeqDesc);
+fail_add_token_types_tuple:
+	Py_DECREF(&rpnif_params_SeqDesc);
+fail_add_iter_expr_params:
+fail_add_ifs:
+	Py_DECREF(&RPNIFSType);
+fail_ifs_type_ready:
+fail_add_iter:
+	Py_DECREF(&RPNIterExprType);
+fail_iter_type_ready:
+fail_add_rpnexpr:
+	Py_DECREF(&RPNExprType);
+fail_expr_type_ready:
+fail_add_tokens:
+	Py_DECREF(tokens_mod);
+fail_add_const:
+	Py_DECREF(const_mod);
+fail_init:
+	Py_DECREF(mod);
+	return NULL;
 }
 
 PyObject* pyrpn_ops(PyObject* mod, PyObject* noargs)
@@ -145,7 +244,7 @@ PyObject* pyrpn_random(PyObject *mod, PyObject *args, PyObject *kwds)
 		return NULL;
 	}
 	
-	expr = rpn_random(expr_sz, args_count);
+	expr = expr_sz?rpn_random(expr_sz, args_count):"";
 	if(!expr)
 	{
 		snprintf(err_str, 128,
@@ -155,6 +254,5 @@ PyObject* pyrpn_random(PyObject *mod, PyObject *args, PyObject *kwds)
 		return NULL;
 	}
 	res = Py_BuildValue("s", expr);
-	//free(expr);
 	return res;
 }

python_pyrpn.h

@@ -28,12 +28,25 @@
 #include "structmember.h"
 
 #include "rpn_jit.h"
+#include "python_const.h"
+#include "python_rpntoken.h"
 #include "python_rpnexpr.h"
 #include "python_if.h"
-#include "python_const.h"
+#include "python_ifs.h"
 
-/**@defgroup python_ext Python API
- * @brief Python API definitions
+/**@file python_pyrpn.h
+ * @brief Python module & type headers
+ * @ingroup python_ext
+ * @ingroup pymod_pyrpn
+ *
+ * This file is the header of the pyrpn Python module definition
+ */
+
+/**@defgroup python_ext Python extension
+ * @brief RPNIFS python extension
+ *
+ * RPNIFS comes with a python extension module allowing to use the RPN
+ * expressions in python.
  *
  * @ref python_pyrpn.c and @ref python_rpnexpr.c should be compiled in a .so file
  * in order to expose a pyrpn Python module.
@@ -48,35 +61,33 @@
  * - pyrpn.RPNExpr.reset_stack(self) to set all stack items to 0
  */
 
-/**@defgroup python_module pyrpn Python module
- * @brief Exposed Python module : pyrpn
+/**@defgroup pymod_pyrpn pyrpn Python module
+ * @brief Python extension main entry-point
  * @ingroup python_ext
  */
 
-/**@file python_pyrpn.h
- * @brief Python module & type headers
- * @ingroup python_module
- *
- * This file is the header of the pyrpn Python module definition
- */
-
+/**@brief RPNExpr Python class type definition */
 extern PyTypeObject RPNExprType;
 
 /**@brief Python module initialization function
- * @ingroup python_module */
+ * @return The initialized module
+ * @ingroup pymod_pyrpn */
 PyMODINIT_FUNC PyInit_pyrpn(void);
-/**@brief pyrpn module methods list
- * @ingroup python_module */
-extern PyMethodDef rpnmodule_methods[];
+
 /**@brief Python module specs
- * @ingroup python_module */
+ * @ingroup pymod_pyrpn */
 extern PyModuleDef rpnmodule;
 
+/**@brief Points on Python's std mmap module */
+extern PyObject *mmap_module;
+/**@brief Python's mmap.mmap class */
+extern PyObject *mmap_cls;
+
 /**@brief Return a dict with valid operations (short keys and long values)
  * @param mod pyrpn module object
  * @param noargs Dummy argument for METH_NOARG
  * @return The a new dict
- * @ingroup python_module
+ * @ingroup pymod_pyrpn
  */
 PyObject* pyrpn_ops(PyObject* mod, PyObject* noargs);
 
@@ -84,7 +95,8 @@ PyObject* pyrpn_ops(PyObject* mod, PyObject* noargs);
  * @param mod pyrpn module object
  * @param args Position arguments in Python list
  * @param kwds Keywords arguments in Python dict
- * @ingroup python_module
+ * @return A str instance
+ * @ingroup pymod_pyrpn
  */
 PyObject* pyrpn_random(PyObject *mod, PyObject *args, PyObject *kwds);
 

python_rpnexpr.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020 Weber Yann
+ * Copyright (C) 2020,2023 Weber Yann
  * 
  * This file is part of pyrpn.
  * 
@@ -18,64 +18,89 @@
  */
 #include "python_rpnexpr.h"
 
-PyMethodDef RPNExpr_methods[] = {
-	{"eval", (PyCFunction)rpnexpr_eval, METH_FASTCALL, "Evaluate an expression"},
-	{"reset_stack", (PyCFunction)rpnexpr_reset_stack, METH_NOARGS,
-		"Reset stack memory storage (set all items to 0)"},
-	{"__getstate__", (PyCFunction)rpnexpr_getstate, METH_NOARGS,
-		"Pickling method. Return a bytes repr of tokenized expression \
-and the stack state."},
-	{"__setstate__", (PyCFunction)rpnexpr_setstate, METH_O,
-		"Unpickling method"},
-	{"uid", (PyCFunction)rpnexpr_getexprstate, METH_NOARGS,
-		"Return a base64 uid for expression"},
+/**@file python_rpnexpr.c
+ * @brief Python module & type definition
+ * @ingroup python_ext
+ * @ingroup python_pyrpn_RPNExpr
+ */
+
+/**@brief @ref pymod_pyrpn_RPNExpr methods definition
+ * @ingroup python_pyrpn_RPNExpr */
+static PyMethodDef RPNExpr_methods[] = {
+	PYRPN_method("random", rpnexpr_random,
+			METH_CLASS | METH_VARARGS | METH_KEYWORDS,
+			"cls, args_count, token_count=10",
+			"Return a new random RPN expression string"),
+	PYRPN_method("default_mutation_params", rpnexpr_default_mutation_params,
+			METH_CLASS | METH_FASTCALL,
+			"cls, /",
+			"Return the default mutation parameters"),
+	PYRPN_method("eval", rpnexpr_eval, METH_FASTCALL,
+			"self, /, *args",
+			"Evaluate an expression"),
+	PYRPN_method("mutate", rpnexpr_mutate,
+			METH_VARARGS | METH_KEYWORDS,
+			"self, n_mutations=1, params=None",
+			"Mutate an expression"),
+	PYRPN_method("reset_stack", rpnexpr_reset_stack,
+			METH_NOARGS,
+			"self, /",
+			"Reset the stack (set all items to 0)"),
+	PYRPN_method("__getstate__", rpnexpr_getstate,
+			METH_NOARGS,
+			"self, /",
+			"Pickling method (see pickle module).\n"
+			"Return a bytes representation of the expression state."),
+	PYRPN_method("__setstate__", rpnexpr_setstate,
+			METH_O,
+			"self, state, /",
+			"Unpickling method (see pickle module)."),
+	PYRPN_method("__copy__", rpnexpr_copy,
+			METH_NOARGS,
+			"self, /",
+			"Return a new equivalent instance (see copy module)"),
+	PYRPN_method("uid", rpnexpr_getexprstate,
+			METH_NOARGS,
+			"self, /",
+			"Return a base64 uid for expression"),
 	{NULL} //Sentinel
 };
 
-PyMemberDef RPNExpr_members[] = {
+/**@brief @ref pymod_pyrpn_RPNExpr members definition
+ * @ingroup python_pyrpn_RPNExpr */
+static PyMemberDef RPNExpr_members[] = {
 	{NULL}
 };
 
+/**@brief @ref pymod_pyrpn_RPNExpr methods definition
+ * @todo Continue sequence implementation with contains, concat, repeat etc.
+ * @ingroup python_pyrpn_RPNExpr */
+static PySequenceMethods RPNExpr_seq_methods = {
+	.sq_length = rpnexpr_len,
+	.sq_item = rpnexpr_token_item,
+	.sq_ass_item = rpnexpr_token_ass_item,
+};
+
+
 PyTypeObject RPNExprType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
-	"pyrpn.RPNExpr",                     /* tp_name */
-	sizeof(PyRPNExpr_t),                        /* tp_basicsize */
-	0,                                               /* tp_itemsize */
-	(destructor)rpnexpr_del, /* tp_dealloc */
-	0,                                               /* tp_print */
-	0,                                               /* tp_getattr */
-	0,                                               /* tp_setattr */
-	0,                                               /* tp_reserved */
-	rpnexpr_repr,                                    /* tp_repr */
-	0,                                               /* tp_as_number */
-	0,                                               /* tp_as_sequence */
-	0,                              /* tp_as_mapping */
-	0,                                               /* tp_hash  */
-	0,                                               /* tp_call */
-	rpnexpr_str,                                               /* tp_str */
-	0,                                               /* tp_getattro */
-	0,                                               /* tp_setattro */
-	0,                                               /* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT |
-	Py_TPFLAGS_BASETYPE,   /* tp_flags */
-	"RPN expression evaluator",                /* tp_doc */
-	0,                                               /* tp_traverse */
-	0,                                               /* tp_clear */
-	0,                                               /* tp_richcompare */
-	0,                                               /* tp_weaklistoffset */
-	0,                                               /* tp_iter */
-	0,                                               /* tp_iternext */
-	RPNExpr_methods,                        /* tp_methods */
-	RPNExpr_members,                        /* tp_members */
-	0,                                               /* tp_getset */
-	0,                                               /* tp_base */
-	0,                                               /* tp_dict */
-	0,                                               /* tp_descr_get */
-	0,                                               /* tp_descr_set */
-	0,                                               /* tp_dictoffset */
-	rpnexpr_init,          /* tp_init */
-	0,                                               /* tp_alloc */
-	rpnexpr_new,                            /* tp_new */
+	.tp_name = "pyrpn.RPNExpr",
+	.tp_doc = "RPN expression evaluator\n\
+\n\
+Instanciation :\n\
+RPNExpr(expression:str, args_count:int, stack_size:int=16)",
+	.tp_basicsize = sizeof(PyRPNExpr_t),
+	.tp_itemsize = 0,
+	.tp_del = rpnexpr_del,
+	.tp_repr = rpnexpr_repr,
+	.tp_str = rpnexpr_str,
+	.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+	.tp_richcompare = rpnexpr_richcompare,
+	.tp_methods = RPNExpr_methods,
+	.tp_as_sequence = &RPNExpr_seq_methods,
+	.tp_members = RPNExpr_members,
+	.tp_init = rpnexpr_init,
+	.tp_new = rpnexpr_new,
 };
 
 PyObject* rpnexpr_new(PyTypeObject *subtype, PyObject *args, PyObject* kwds)
@@ -91,6 +116,7 @@ PyObject* rpnexpr_new(PyTypeObject *subtype, PyObject *args, PyObject* kwds)
 	expr = (PyRPNExpr_t*)ret;
 	expr->rpn = NULL;
 	expr->args = NULL;
+	expr->borrowed_expr = 0;
 	return ret;
 }
 
@@ -113,13 +139,6 @@ int rpnexpr_init(PyObject *self, PyObject *args, PyObject *kwds)
 		return -1;
 	}
 
-	if(strlen(expr) == 0)
-	{
-		PyErr_SetString(PyExc_ValueError,
-			"RpnExpr.__init__() expect expression argument to be not empty");
-		return -1;
-	}
-
 	if(args_count < 0 || args_count > 255)
 	{
 		snprintf(err_str, 128,
@@ -174,12 +193,44 @@ int rpnexpr_init(PyObject *self, PyObject *args, PyObject *kwds)
 	return 0;
 }
 
+PyObject* rpnexpr_init_borrowing(rpn_expr_t *borrowed)
+{
+	PyObject *args, *ret;
+	PyRPNExpr_t *instance;
+
+	args  = Py_BuildValue("sLL", "", borrowed->args_count,
+			borrowed->stack_sz);
+	if(!args || PyErr_Occurred())
+	{
+		return NULL;
+	}
+
+	ret = PyObject_CallObject((PyObject*)&RPNExprType, args);
+	if(!ret || PyErr_Occurred())
+	{
+		Py_DECREF(args);
+		return NULL;
+	}
+
+	Py_DECREF(args);
+
+	instance = (PyRPNExpr_t*)ret;
+
+	rpn_expr_close(instance->rpn);
+	free(instance->rpn);
+
+	instance->borrowed_expr = 1;
+	instance->rpn = borrowed;
+	
+	return ret;
+}
+
 void rpnexpr_del(PyObject *self)
 {
 	PyRPNExpr_t *expr_self;
 
 	expr_self = (PyRPNExpr_t*)self;
-	if(expr_self->rpn)
+	if(expr_self->rpn && !expr_self->borrowed_expr)
 	{
 		rpn_expr_close(expr_self->rpn);
 		free(expr_self->rpn);
@@ -195,48 +246,16 @@ void rpnexpr_del(PyObject *self)
 
 PyObject* rpnexpr_getexprstate(PyObject *self, PyObject *noargs)
 {
-	/*
-	PyObject *base64_mod, *base64_encode, *gzip, *compress;
-	PyObject *bytes_repr, *comp, *res;
-	if(!(base64_mod = PyImport_ImportModule("base64")))
-	{
-		return NULL;
-	}
-	if(!(base64_encode = PyObject_GetAttrString(base64_mod, "b64encode")))
-	{
-		return NULL;
-	}
-	if(!(gzip = PyImport_ImportModule("gzip")))
-	{
-		return NULL;
-	}
-	if(!(compress = PyObject_GetAttrString(gzip, "compress")))
-	{
-		return NULL;
-	}
-
-	bytes_repr = _rpnexpr_getstate(self, noargs, 0);
-
-	res = PyObject_CallOneArg(base64_encode, bytes_repr);
-	Py_DECREF(bytes_repr);
-	return res;
-
-	comp = PyObject_CallOneArg(compress, bytes_repr);
-	Py_DECREF(bytes_repr);
-	
-	res = PyObject_CallOneArg(base64_encode,  comp);
-	Py_DECREF(comp);
-	return res;
-	*/
+	/**@todo check if usefull */
+	PyErr_SetString(PyExc_NotImplementedError, "Not implemented...");
 	return NULL;
 }
 
 PyObject* rpnexpr_getstate(PyObject *self, PyObject *noargs)
 {
-	PyObject *res, *part;
-	PyRPNExpr_state_t resbuf;
+	PyObject *res;
 	PyRPNExpr_t *expr_self;
-	size_t total_sz, i;
+	size_t total_sz;
 	char err_str[128];
 
 	expr_self = (PyRPNExpr_t*)self;
@@ -250,74 +269,26 @@ PyObject* rpnexpr_getstate(PyObject *self, PyObject *noargs)
 		return NULL;
 	}
 
-	total_sz = sizeof(PyRPNExpr_state_t);
-	total_sz += expr_self->rpn->toks.tokens_sz * sizeof(rpn_token_t);
-	total_sz += expr_self->rpn->stack_sz * sizeof(rpn_value_t);
+	total_sz = rpn_expr_serialize(expr_self->rpn, NULL, 0);
 
-	resbuf.total_sz = total_sz;
-	resbuf.argc = expr_self->rpn->args_count;
-	resbuf.stack_sz = expr_self->rpn->stack_sz;
-	resbuf.token_sz = expr_self->rpn->toks.tokens_sz;
+	char serial[total_sz];
 
-	if(!(res = PyBytes_FromStringAndSize((char*)&resbuf, sizeof(resbuf))))
+	rpn_expr_serialize(expr_self->rpn, serial, total_sz);
+	if(!(res = PyBytes_FromStringAndSize(serial, total_sz)))
 	{
 		return NULL;
 	}
-	if(resbuf.stack_sz)
-	{
-		if(!(part=PyBytes_FromStringAndSize(
-				(char*)expr_self->rpn->stack,
-				sizeof(rpn_value_t) * resbuf.stack_sz)))
-		{
-			return NULL;
-		}
-		PyBytes_ConcatAndDel(&res, part);
-		if(!res)
-		{
-			return NULL;
-		}
-	}
-	if(resbuf.token_sz)
-	{
-		for(i=0; i<expr_self->rpn->toks.tokens_sz;i++)
-		{
-			// restore op pointers
-			expr_self->rpn->toks.tokens[i].op = NULL;
-		}
-	
-		if(!(part=PyBytes_FromStringAndSize(
-				(char*)expr_self->rpn->toks.tokens,
-				sizeof(rpn_token_t) * resbuf.token_sz)))
-		{
-			return NULL;
-		}
-		for(i=0; i<expr_self->rpn->toks.tokens_sz;i++)
-		{
-			// restore op pointers
-			expr_self->rpn->toks.tokens[i].op = &(rpn_ops[expr_self->rpn->toks.tokens[i].op_n]);
-		}
-	
-		PyBytes_ConcatAndDel(&res, part);
-		if(!res)
-		{
-			return NULL;
-		}
-	}
 	return res;
 }
 
 PyObject* rpnexpr_setstate(PyObject *self, PyObject *state_bytes)
 {
 	PyObject *tmp, *tmp2;
-	PyRPNExpr_state_t *state;
 	PyRPNExpr_t *expr_self;
-	rpn_value_t *stack;
-	rpn_tokenized_t toks;
 	const char *data;
-	size_t bsize, csize;
+	size_t bsize;
 	int err;
 	char err_str[256];
-	size_t i;
 
 	expr_self = (PyRPNExpr_t*)self;
 	
@@ -348,7 +319,7 @@ generating exception message !");
 	if(expr_self->rpn || expr_self->args) /* checking instance state */
 	{
 		PyErr_SetString(PyExc_ValueError,
-			"RPNExpr.__getstate__() instance in bad state : \
+			"RPNExpr.__setstate__() instance in bad state : \
 should not be initialized");
 		return NULL;
 	}
@@ -356,37 +327,14 @@ should not be initialized");
 	/* Checking data size */
 	bsize = PyBytes_GET_SIZE(state_bytes);
 	data = PyBytes_AS_STRING(state_bytes);
-	state = (PyRPNExpr_state_t*)data;
 
 	if(bsize < sizeof(size_t))
 	{
 		PyErr_SetString(PyExc_ValueError, "Invalid argument");
-	}
-	if(bsize != state->total_sz)
-	{
-		snprintf(err_str, 128,
-			"RPNExpr.__setstate__() error : expected state to \
-contains %ld bytes but %ld found",
-			state->total_sz, bsize);
-		PyErr_SetString(PyExc_ValueError, err_str);
-	}
-
-	/* Checking data "integrity" */
-	csize = sizeof(PyRPNExpr_state_t) +
-		state->token_sz * sizeof(rpn_token_t) +
-		state->stack_sz * sizeof(rpn_value_t);
-	if(state->total_sz != csize)
-	{
-		snprintf(err_str, 128,
-			"RPNExpr.__setstate__() error : should have %ld as \
-total size but %ld found",
-			csize, state->total_sz);
-		PyErr_SetString(PyExc_ValueError, err_str);
 		return NULL;
 	}
 
-
-	/* Alloc and init rpn expression & args buffer*/
+	//alloc & init
 	if(!(expr_self->rpn = malloc(sizeof(rpn_expr_t))))
 	{
 		err = errno;
@@ -398,7 +346,16 @@ rpn_expr_t : %s",
 		return NULL;
 	}
 	bzero(expr_self->rpn, sizeof(rpn_expr_t));
-	if(!(expr_self->args = malloc(sizeof(rpn_value_t)*state->argc)))
+
+	if(rpn_expr_deserialize(expr_self->rpn, data, bsize) < 0)
+	{
+		PyErr_Format(PyExc_ValueError,
+				"RPNExpr.__setstate__() fails to deserialize (%s)) %s",
+			    	strerror(errno), expr_self->rpn->err_reason);
+		return NULL;
+	}
+
+	if(!(expr_self->args = malloc(sizeof(rpn_value_t)*expr_self->rpn->args_count)))
 	{
 		err = errno;
 		snprintf(err_str, 128, 
@@ -409,60 +366,138 @@ args buffer : %s",
 		return NULL;
 	}
 
-	if(rpn_expr_init(expr_self->rpn, state->stack_sz, state->argc) < 0)
-	{
-		snprintf(err_str, 256,
-			"RPNExpr.__setstate__() failed to init RPN expression \
-: %s",
-			expr_self->rpn->err_reason);
-		PyErr_SetString(PyExc_RuntimeError, err_str);
-		goto free_err;
-	}
-
-	/* restore stack & untokenize expression */
-	stack = (rpn_value_t*)(state+1);
-	memcpy(expr_self->rpn->stack, stack,
-		sizeof(rpn_value_t)*state->stack_sz);
-
-	toks.argc = state->argc;
-	toks.tokens_sz = state->token_sz;
-	toks.tokens = malloc(sizeof(rpn_token_t)*state->token_sz);
-	if(!toks.tokens)
-	{
-		snprintf(err_str, 128,
-			"RPNExpr.__setstate__() failed to allocate tokens \
-: %s",
-			strerror(errno));
-		PyErr_SetString(PyExc_RuntimeError, err_str);
-		goto close_err;
-	}
-	memcpy(toks.tokens, (rpn_token_t*)(stack+state->stack_sz),
-		sizeof(rpn_token_t)*state->token_sz);
-
-	for(i=0; i<toks.tokens_sz;i++)
-	{
-		// restore op pointers
-		toks.tokens[i].op = &(rpn_ops[toks.tokens[i].op_n]);
-	}
-	
-	if(rpn_expr_untokenize(expr_self->rpn, &toks, 0) < 0)
-	{
-		snprintf(err_str, 256,
-			"RPNExpr.__setstate__() unable to untokenize : %s",
-			expr_self->rpn->err_reason);
-		PyErr_SetString(PyExc_RuntimeError, err_str);
-		goto close_err;
-	}
-	expr_self->rpn->toks = toks;
-	
-
 	Py_RETURN_NONE;
-	close_err:
-		rpn_expr_close(expr_self->rpn);
-	free_err:
-		return NULL;
 }
 
+
+PyObject* rpnexpr_copy(PyObject *self, PyObject *noargs)
+{
+	PyRPNExpr_t *copy;
+	PyObject *ret, *state, *setret;
+
+	ret = PyObject_CallMethod((PyObject*)&RPNExprType, "__new__", "O", &RPNExprType);
+	copy = (PyRPNExpr_t*)ret;
+
+	state = PyObject_CallMethod(self, "__getstate__", NULL);
+	if(PyErr_Occurred()) {
+		goto err;
+	}
+	setret = PyObject_CallMethod(ret, "__setstate__", "O", state);
+	if(PyErr_Occurred()) {
+		Py_DECREF(state);
+		goto err;
+	}
+
+	Py_DECREF(state);
+	Py_DECREF(setret);
+
+	return ret;
+err:
+	PyObject_Del(copy);
+	return NULL;
+}
+
+
+Py_ssize_t rpnexpr_len(PyObject *self)
+{
+	PyRPNExpr_t *expr_self = (PyRPNExpr_t*)self;
+	return (Py_ssize_t)expr_self->rpn->toks.tokens_sz;
+}
+
+
+PyObject* rpnexpr_token_item(PyObject *self, Py_ssize_t idx)
+{
+	PyRPNExpr_t *expr_self = (PyRPNExpr_t*)self;
+	Py_ssize_t _idx = idx;
+	if(idx < 0)
+	{
+		idx = expr_self->rpn->toks.tokens_sz - 1 + idx;
+	}
+	if(idx < 0 || (size_t)idx >= expr_self->rpn->toks.tokens_sz)
+	{
+		PyErr_Format(PyExc_IndexError,
+			"No token %ld in expression of size %ld",
+			_idx, expr_self->rpn->toks.tokens_sz);
+		return NULL;
+
+	}
+	return rpntoken_from_token(&expr_self->rpn->toks.tokens[idx]);
+}
+
+
+int rpnexpr_token_ass_item(PyObject *self, Py_ssize_t idx, PyObject* elt)
+{
+	PyRPNExpr_t *expr_self = (PyRPNExpr_t*)self;
+	Py_ssize_t _idx = idx;
+	if(idx < 0)
+	{
+		idx = expr_self->rpn->toks.tokens_sz - 1 + idx;
+	}
+	if(idx < 0 || (size_t)idx > expr_self->rpn->toks.tokens_sz)
+	{
+		PyErr_Format(PyExc_IndexError,
+			"Cannot set token %ld in expression of size %ld",
+			_idx, expr_self->rpn->toks.tokens_sz);
+		return -1;
+	}
+
+	if(!PyObject_IsInstance(elt, (PyObject*)&RPNTokenType))
+	{
+		PyErr_Format(PyExc_TypeError,
+				"Given element in not RPNToken subtype");
+		return -1;
+	}
+
+	short new_elt = 0;
+	if((size_t)idx == expr_self->rpn->toks.tokens_sz)
+	{
+		new_elt = 1;
+		expr_self->rpn->toks.tokens_sz++;
+		size_t new_sz = expr_self->rpn->toks.tokens_sz*sizeof(rpn_token_t);
+		rpn_token_t *tmp = realloc(expr_self->rpn->toks.tokens, new_sz);
+		if(!tmp)
+		{
+			PyErr_Format(PyExc_MemoryError,
+					"Error reallocation tokenized expression : %s",
+					strerror(errno));
+			return -1;
+		}
+		expr_self->rpn->toks.tokens = tmp;
+	}
+
+	rpn_token_t original = expr_self->rpn->toks.tokens[idx];
+
+	RPNToken_t *token = (RPNToken_t*)elt;
+	expr_self->rpn->toks.tokens[idx] = token->value;
+	if(rpn_expr_tokens_updated(expr_self->rpn) < 0)
+	{
+		PyErr_Format(PyExc_ValueError,
+				"Unable to update expression : %s",
+				strerror(errno));
+		goto rollback;
+	}
+	return 0;
+
+rollback:
+	if(new_elt)
+	{
+		expr_self->rpn->toks.tokens_sz--;
+	}
+	else
+	{
+		expr_self->rpn->toks.tokens[idx] = original;
+		if(rpn_expr_tokens_updated(expr_self->rpn) < 0)
+		{
+			PyErr_Format(PyExc_RuntimeError,
+					"Unable to rollback expression : %s",
+					strerror(errno));
+			goto rollback;
+		}
+	}
+	return -1;
+}
+
+
 PyObject* rpnexpr_eval(PyObject* self, PyObject** argv, Py_ssize_t argc)
 {
 	PyRPNExpr_t *expr_self;
@@ -504,11 +539,57 @@ PyObject* rpnexpr_eval(PyObject* self, PyObject** argv, Py_ssize_t argc)
 	}
 
 	res = rpn_expr_eval(expr_self->rpn, expr_self->args);
-//dprintf(2, "[RES=%lu]\n", res);
 
 	return PyLong_FromUnsignedLong(res);
 }
 
+
+PyObject* rpnexpr_mutate(PyObject* slf, PyObject *args, PyObject *kwds)
+{
+	PyRPNExpr_t *self = (PyRPNExpr_t*)slf;
+
+	char *str_args = "|IO:RPNExpr.mutate";
+	char *names[] = {"n_mutations", "params", NULL};
+
+	PyObject *py_params = NULL;
+	unsigned int n_mutations = 1;
+	rpn_mutation_params_t params;
+
+	if(!PyArg_ParseTupleAndKeywords(args, kwds, str_args, names,
+				&n_mutations, &py_params))
+	{
+		return NULL;
+	}
+
+	if(!py_params || py_params == Py_None)
+	{
+		if(py_params == Py_None) { Py_DECREF(Py_None); }
+		memcpy(&params, &rpn_mutation_params_default,
+				sizeof(rpn_mutation_params_t));
+	}
+	else
+	{
+		if(rpnexpr_pyobj_to_mutation_params(py_params, &params) < 0)
+		{
+			PyErr_SetString(PyExc_ValueError, "Bad value for params arguments");
+			return NULL;
+		}
+	}
+
+	for(size_t i=0; i<n_mutations; i++)
+	{
+		if(rpn_mutation(&(self->rpn->toks), &params) < 0)
+		{
+			PyErr_Format(PyExc_RuntimeError, "Mutation failed : %s",
+					strerror(errno));
+			return NULL;
+		}
+	}
+
+	rpn_expr_tokens_updated(self->rpn);
+	Py_RETURN_NONE;
+}
+
 PyObject* rpnexpr_reset_stack(PyObject *self, PyObject *noargs)
 {
 	rpn_expr_reset_stack(((PyRPNExpr_t*)self)->rpn);
@@ -553,3 +634,221 @@ PyObject* rpnexpr_repr(PyObject *self)
 	return res;
 }
 
+
+PyObject* rpnexpr_richcompare(PyObject *_self, PyObject *other, int op)
+{
+	PyObject *sta, *stb, *res;
+
+	if(!PyObject_IsInstance(other, (PyObject*)&RPNExprType))
+	{
+		PyErr_Format(PyExc_TypeError,
+				"Can only be compared with RPNExpr");
+		return NULL;
+	}
+
+	sta = rpnexpr_getstate(_self, NULL);
+	stb = rpnexpr_getstate(other, NULL);
+
+	res = PyObject_RichCompare(sta, stb, op);
+
+	Py_DECREF(sta);
+	Py_DECREF(stb);
+
+	return res;
+}
+
+
+PyObject* rpnexpr_random(PyObject *cls, PyObject *args, PyObject *kwds)
+{
+	long long int args_count, expr_sz;
+	char *expr, err_str[128];
+	PyObject *res;
+	char *names[] = {"args_count", "token_count", NULL};
+
+	expr_sz = 10;
+	if(!PyArg_ParseTupleAndKeywords(args, kwds, "L|L:pyrpn.random_expr", names,
+		&args_count, &expr_sz))
+	{
+		return NULL;
+	}
+	
+	expr = rpn_random(expr_sz, args_count);
+	if(!expr)
+	{
+		snprintf(err_str, 128,
+			"Error generating random expression : %s",
+			strerror(errno));
+		PyErr_SetString(PyExc_RuntimeError, err_str);
+		return NULL;
+	}
+	res = Py_BuildValue("s", expr);
+	//free(expr);
+	return res;
+}
+
+PyObject* rpnexpr_default_mutation_params(PyObject *cls, PyObject **argv, Py_ssize_t argc)
+{
+	PyObject *res, *wtypes;
+
+	if(!(wtypes = PyStructSequence_New(&rpn_token_types_SeqDesc)))
+	{
+		return NULL;
+	}
+
+	if(!(res = PyStructSequence_New(&rpn_mutation_params_SeqDesc)))
+	{
+		return NULL;
+	}
+
+	// wtypes filled with 1.0
+	PyObject *one = PyFloat_FromDouble(1.0);
+	for(size_t i=0; i<3; i++)
+	{
+		PyStructSequence_SET_ITEM(wtypes, i, one);
+	}
+
+
+	// max_len
+	PyStructSequence_SET_ITEM(res, 0,
+			PyLong_FromLong(rpn_mutation_params_default.min_len));
+	// weight_add
+	PyStructSequence_SET_ITEM(res, 1,
+			PyFloat_FromDouble(rpn_mutation_params_default.w_add));
+	// weight_del
+	PyStructSequence_SET_ITEM(res, 2,
+			PyFloat_FromDouble(rpn_mutation_params_default.w_del));
+	// weight_mut
+	PyStructSequence_SET_ITEM(res, 3,
+			PyFloat_FromDouble(rpn_mutation_params_default.w_mut));
+	// weight_mut_soft
+	PyStructSequence_SET_ITEM(res, 4,
+			PyFloat_FromDouble(rpn_mutation_params_default.w_mut_soft));
+
+	/** TODO use rpn_mutation_params_default instead of wtypes [1,1,1] */
+	// weight_add_elt
+	PyStructSequence_SET_ITEM(res, 5, wtypes);
+	Py_INCREF(wtypes);
+	// weight_mut_elt
+	PyStructSequence_SET_ITEM(res, 6, wtypes);
+	Py_INCREF(wtypes);
+
+	return res;
+
+}
+
+
+/**@brief Parse given object to float
+ * @param obj The python object to parse
+ * @param res Points on result
+ * @note Raise python exception on error
+ */
+static inline void _parse_float(PyObject *obj, float *res)
+{
+	double tmp = PyFloat_AsDouble(obj);
+	if(tmp > FLT_MAX)
+	{
+		PyErr_SetString(PyExc_OverflowError, "Float overflows");
+		return;
+	}
+	*res=tmp;
+	return;
+}
+
+/** Parse weights python parameter
+ * @param obj The python object
+ * @param w Points on the 3 weights to parse
+ * @note Raise python exception on error
+ */
+static inline void _parse_type_weights(PyObject *obj, float w[3])
+{
+	PyObject *fast = PySequence_Fast(obj, "Not a RPNTokenTypeTuple nor with a length of 3");
+	if(!fast) { return; }
+	if(PySequence_Length(obj) != 3)
+	{
+		PyErr_Format(PyExc_ValueError, "Excpected RPNTokenTypeTuple or 3 elements but got %d elements", PySequence_Length(obj));
+		return;
+	}
+
+	PyObject **elts = PySequence_Fast_ITEMS(fast);
+
+	const char* names[3] = {"op", "const", "var"};
+	for(size_t i=0; i<3; i++)
+	{
+		_parse_float(elts[i], &w[i]);
+		if(PyErr_Occurred())
+		{
+			PyErr_Format(PyExc_ValueError,
+					"Bad value for .%s field",
+					names[i]);
+			break;
+		}
+	}
+	return;
+}
+
+int rpnexpr_pyobj_to_mutation_params(PyObject* py_params, rpn_mutation_params_t *params)
+{
+	if(!PySequence_Check(py_params))
+	{
+		PyErr_SetString(PyExc_TypeError, "The given object is not a a sequence");
+		return -1;
+	}
+	if(PySequence_Size(py_params) != 7)
+	{
+		PyErr_SetString(PyExc_ValueError, "The given object is not a RPNMutationParamsTuple nor of length 7");
+		return -1;
+	}
+	PyObject *fast = PySequence_Fast(py_params, "The given object is not a RPNMutationParamsTuple nor a sequence ?");
+	Py_INCREF(fast);
+
+	PyObject **elts = PySequence_Fast_ITEMS(fast);
+
+	params->min_len = PyLong_AsSize_t(elts[0]);
+	if(PyErr_Occurred())
+	{
+		PyErr_SetString(PyExc_ValueError, "Bad type for .minimum_length field");
+		return -1;
+	}
+
+	_parse_float(elts[1], &params->w_add);
+	if(PyErr_Occurred())
+	{
+		PyErr_SetString(PyExc_ValueError, "Bad value for .weight_add field");
+		return -1;
+	}
+	_parse_float(elts[2], &params->w_del);
+	if(PyErr_Occurred())
+	{
+		PyErr_SetString(PyExc_ValueError, "Bad value for .weight_del field");
+		return -1;
+	}
+	_parse_float(elts[3], &params->w_mut);
+	if(PyErr_Occurred())
+	{
+		PyErr_SetString(PyExc_ValueError, "Bad value for .weight_mut field");
+		return -1;
+	}
+	_parse_float(elts[4], &params->w_mut_soft);
+	if(PyErr_Occurred())
+	{
+		PyErr_SetString(PyExc_ValueError, "Bad value for .weight_mut_soft field");
+		return -1;
+	}
+	
+	_parse_type_weights(elts[5], params->w_add_elt);
+	if(PyErr_Occurred())
+	{
+		PyErr_SetString(PyExc_ValueError, "Bad value for .weight_add_elt field");
+		return -1;
+	}
+
+	_parse_type_weights(elts[6], params->w_mut_elt);
+	if(PyErr_Occurred())
+	{
+		PyErr_SetString(PyExc_ValueError, "Bad value for .weight_add_elt field");
+		return -1;
+	}
+	rpn_mutation_init_params(params);
+	return 0;
+}
+

python_rpnexpr.h

@@ -22,39 +22,39 @@
 #include "config.h"
 
 #include <errno.h>
+#include <float.h>
 
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include "structmember.h"
 
 #include "rpn_jit.h"
-
-/**@defgroup python_type RPNExpr Python class
- * @brief Exposed Python class : RPNExpr
- * @ingroup python_module
- */
+#include "python_rpntoken.h"
+#include "python_const.h"
 
 /**@file python_rpnexpr.h
- * @brief Python RPNExpr type headers
- * @ingroup python_type
+ * @brief pyrpn.RPNExpr definition
+ * @ingroup python_ext
+ * @ingroup pymod_pyrpn_RPNExpr
  *
  * This file is the header of the RPNExpr Python class
  */
 
-/**@brief RPNExpr Python class methods list
- * @ingroup python_type */
-extern PyMethodDef RPNExpr_methods[];
-/**@brief RPNExpr Python class members list
- * @ingroup python_type */
-extern PyMemberDef RPNExpr_members[];
+/**@defgroup pymod_pyrpn_RPNExpr pyrpn.RPNExpr
+ * @brief Exposed Python class : RPNExpr
+ * @ingroup pymod_pyrpn
+ */
+
+
 /**@brief RPNExpr Python class type definition
- * @ingroup python_type */
+ * @ingroup pymod_pyrpn_RPNExpr */
 extern PyTypeObject RPNExprType;
 
 /**@brief Structure holding RPNExpr objects
- * @ingroup python_type */
+ * @ingroup pymod_pyrpn_RPNExpr */
 typedef struct
 {
+	/** Python's type definition */
 	PyObject_VAR_HEAD;
 	
 	/**@brief Pointer on @ref rpn_expr_s */
@@ -63,24 +63,43 @@ typedef struct
 	/**@brief Array storing expression argument
 	 * @note As attribute of rpn_expr allowing malloc & free only once */
 	rpn_value_t *args;
+
+	/**@brief If true, someone else will take care of freeing the
+	 * rpn expression at deletion */
+	short borrowed_expr;
 } PyRPNExpr_t;
 
 /**@brief Organize PyRPNExpr_t state data
  * @see rpnexpr_getstate
- * @see rpnexpr_setstate */
+ * @see rpnexpr_setstate 
+ * @ingroup pymod_pyrpn_RPNExpr */
 typedef struct
 {
+	/**@brief The total size of the state */
 	size_t total_sz;
+	/**@brief Expression's argument count */
 	size_t argc;
+	/**@brief Expression's stack size */
 	unsigned char stack_sz;
+	/**@brief Number of tokens in expression */
 	size_t token_sz;
 } PyRPNExpr_state_t;
 
+/**@brief Return a new instance of RPNExpr with a borrowed expression
+ * @param borrowed An rpn expression to borrow
+ * @note Borrowed expression means that the rpn_expr_t is handled by someone
+ *       else and should not be freed when the instance is deleted
+ * @return A new RPNExpr instance
+ * @ingroup pymod_pyrpn_RPNExpr 
+ */
+PyObject* rpnexpr_init_borrowing(rpn_expr_t *borrowed);
+
 /**@brief RpnExpr __new__ method
  * @param subtype Type of object being created (pyrpn.RPNExpr)
  * @param args positional arguments for subtype
- * @param kwargs keyword argumenrs for subtype
+ * @param kwds keyword argumenrs for subtype
  * @return The new Python RPNExpr object
+ * @ingroup pymod_pyrpn_RPNExpr 
  */
 PyObject* rpnexpr_new(PyTypeObject *subtype, PyObject *args, PyObject* kwds);
 
@@ -89,13 +108,13 @@ PyObject* rpnexpr_new(PyTypeObject *subtype, PyObject *args, PyObject* kwds);
  * @param args Positional arguments list
  * @param kwds Keywords arguments dict
  * @return 0 if no error else -1
- * @ingroup python_type
+ * @ingroup pymod_pyrpn_RPNExpr
  */
 int rpnexpr_init(PyObject *self, PyObject *args, PyObject *kwds);
 
 /**@brief RPNExpr __del__ method 
  * @param self RPNExpr instance
- * @ingroup python_type
+ * @ingroup pymod_pyrpn_RPNExpr
  */
 void rpnexpr_del(PyObject *self);
 
@@ -104,6 +123,7 @@ void rpnexpr_del(PyObject *self);
  * @param self RPNExpr instance
  * @param noargs Not an argument...
  * @return A bytes Python instance
+ * @ingroup pymod_pyrpn_RPNExpr 
  */
 PyObject* rpnexpr_getexprstate(PyObject *self, PyObject *noargs);
 
@@ -112,6 +132,7 @@ PyObject* rpnexpr_getexprstate(PyObject *self, PyObject *noargs);
  * @param noargs Not an argument...
  * @return A bytes Python instance suitable as argument for
  * @ref rpnexpr_setstate
+ * @ingroup pymod_pyrpn_RPNExpr 
  */
 PyObject* rpnexpr_getstate(PyObject *self, PyObject *noargs);
 
@@ -121,37 +142,117 @@ PyObject* rpnexpr_getstate(PyObject *self, PyObject *noargs);
  * rpnexpr_getstate
  * @return A bytes Python instance suitable as argument for
  * @ref rpnexpr_getstate
+ * @ingroup pymod_pyrpn_RPNExpr 
  */
 PyObject* rpnexpr_setstate(PyObject *cls, PyObject *state);
 
+/**@brief RPNExpr __copy__ method for cloning
+ * @param cls RPNExpr class
+ * @param noargs Not an argument...
+ * @return A new cloned instance
+ * @ref rpnexpr_setstate
+ * @ingroup pymod_pyrpn_RPNExpr 
+ */
+PyObject* rpnexpr_copy(PyObject *cls, PyObject *noargs);
+
+/**@brief RPNExpr.__len__() method
+ * @param self RPNExpr instance
+ * @return A integer with the number of tokens in expression
+ * @ingroup pymod_pyrpn_RPNExpr 
+ */
+Py_ssize_t rpnexpr_len(PyObject *self);
+
+/**@brief Sequence method for __getitem__ token getter
+ * @param self RPNExpr instance
+ * @param idx The item index (can be negative)
+ * @return A @ref RPNTokenType subclass instance
+ * @ingroup pymod_pyrpn_RPNExpr 
+ */
+PyObject* rpnexpr_token_item(PyObject *self, Py_ssize_t idx);
+
+/**@brief Sequence method for __setitem__ token setter
+ * @param self RPNExpr instance
+ * @param idx The item index
+ * @param elt An RPNTokenType subclass token to set
+ * @return 0 or -1 on error and raise IndexError
+ * @ingroup pymod_pyrpn_RPNExpr 
+ */
+int rpnexpr_token_ass_item(PyObject *self, Py_ssize_t idx, PyObject* elt);
+
 /**@brief Eval an RPN expression given arguments and return the
  * value
  * @param self RPNExpr instance
  * @param argv Array of PyObject* arguments
  * @param argc Number of arguments
  * @return The value resulting of evaluation
- * @ingroup python_type
+ * @ingroup pymod_pyrpn_RPNExpr
  */
 PyObject* rpnexpr_eval(PyObject* self, PyObject** argv, Py_ssize_t argc);
 
+/**@brief Mutate an RPN expression given arguments and return the value
+ * @param self RPNExpr instance
+ * @param args Positionnal arguments
+ * @param kwds Keyword arguments
+ * @return Py_None
+ * @ingroup pymod_pyrpn_RPNExpr 
+ */
+PyObject* rpnexpr_mutate(PyObject* self, PyObject *args, PyObject *kwds);
+
 /**@brief Set all stack item to zero
  * @param self RPNExpr instance
  * @param noargs Dummy argument for METH_NOARG
  * @return None
- * @ingroup python_type
+ * @ingroup pymod_pyrpn_RPNExpr
  */
 PyObject* rpnexpr_reset_stack(PyObject *self, PyObject *noargs);
 
 /**@brief RPNExpr.__repr__()
  * @param self RPNExpr instance
- * @ingroup python_type
+ * @return An str instance representing the exppression
+ * @ingroup pymod_pyrpn_RPNExpr
  */
 PyObject* rpnexpr_repr(PyObject *self);
 
 /**@brief RPNExpr.__str__()
  * @param self RPNExpr instance
- * @ingroup python_type
+ * @return An str instance representing the exppression
+ * @ingroup pymod_pyrpn_RPNExpr
  */
 PyObject* rpnexpr_str(PyObject *self);
 
+/**@brief RPNExpr PEP-207 richcompare method
+ * @param _self RPNExpr instance
+ * @param other RPNExpr instance to compare to
+ * @param op The comparison to be done
+ * @return Py_True or Py_False
+ * @ingroup pymod_pyrpn_RPNExpr 
+ */
+PyObject* rpnexpr_richcompare(PyObject *_self, PyObject *other, int op);
+
+/**@brief Return a new Python str with a random RPN expression
+ * @param cls pyrpn module object
+ * @param args Position arguments in Python list
+ * @param kwds Keywords arguments in Python dict
+ * @return A str instance
+ * @ingroup pymod_pyrpn
+ */
+PyObject* rpnexpr_random(PyObject *cls, PyObject *args, PyObject *kwds);
+
+/**@brief Return a new named tuple containing default mutation parameters
+ * @param cls The class (class method)
+ * @param argv The arguments (FASTCALL)
+ * @param argc The number of arguments
+ * @return The named tuple
+ * @ingroup pymod_pyrpn_RPNExpr 
+ */
+PyObject* rpnexpr_default_mutation_params(PyObject *cls, PyObject **argv, Py_ssize_t argc);
+
+/**@brief Try to convert a python object into a rpn_mutation_params_t
+ * @param py_params The python object
+ * @param params A pointer on the parameters to initialize
+ * @return -1 on failure and raise an exception
+ * @ingroup pymod_pyrpn_RPNExpr 
+ */
+int rpnexpr_pyobj_to_mutation_params(PyObject* py_params, rpn_mutation_params_t *params);
+
 #endif

python_rpnifs/__main__.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 import copy
+import random
+import multiprocessing
 
 try:
     import pyrpn
@@ -19,16 +21,24 @@ from .ifs3 import IFS3
 from .ifs4 import IFS4
 from .ifs5 import IFS5
 
+from .shape_score import ShapeScore
+
 
 #steps = 1024**2
-#steps=512**2
-steps=1024**2
+steps=512**2
+#steps=1024**2
 #steps*=4
 #steps*=10
-pool_sz = 30
-best_sz = 6
+
+#pool_sz = 30
+#best_sz = 6
+
+pool_sz = 32
+best_sz = 8
+
 n_mutation = 10
 outfile='/tmp/rpnifs2_%02d.png'
+outfile_grp='/tmp/rpnifs2_%02d_grp.png'
 #world = IFS1.get_world()
 #pool = [IFS1(init_sz=3, world=world) for _ in range(pool_sz)]
 #world = IFS2.get_world()
@@ -42,7 +52,18 @@ world = IFS5.get_world()
 pool = [IFS5(init_sz=6, world=world) for _ in range(pool_sz)]
 print('POOL ready')
 
+def stepper(ifs):
+    for _ in range(steps):
+        ifs.step()
+    return ifs
+
+def get_score(world):
+    shapescore = ShapeScore(world, jobs=1)
+    score = shapescore.score()
+    return score, shapescore
+
 generation=0
+mp_pool = multiprocessing.Pool(2)
 while True:
     generation+=1
     print('='*78)
@@ -52,18 +73,27 @@ while True:
     scores = []
     i=0
     #for ifs in pool:
-    for ifs in tqdm(pool, unit='ifs'):
-        i+=1
-        ifs.raz_world()
-        #for _ in tqdm(range(steps), total=steps, unit_scale=True):
-        for _ in range(steps):
-            ifs.step()
+    #for ifs in tqdm(pool, unit='ifs'):
+    #    ifs.raz_world()
+    #    #for _ in tqdm(range(steps), total=steps, unit_scale=True):
+    #    for _ in range(steps):
+    #        ifs.step()
+    pool = [ifs for ifs in tqdm(mp_pool.imap(stepper, pool),
+                                total=len(pool),
+                                unit='ifs')]
+    
+    for i, (score,shapescore) in tqdm(enumerate(mp_pool.imap(get_score,
+                                                             [ifs._world for ifs in pool])),
+                                      unit='ifs', total=len(pool)):
+        ifs = pool[i]
 
-        score = ifs.score()
+        #score = ifs.score()
         scores.append((ifs, score))
-        print('%02d) %5.3f %s' % (i, score, ifs))
+        #print('%02d) %5.3f %s' % (i, score, ifs))
+        print('%02d) %5.3f' % (i, score))
         im = Image.fromarray(ifs.get_image())
         im.save(outfile % i)
+        shapescore.result_image(outfile_grp % i)
 
         im_png = PngImageFile(outfile % i)
         metadata = PngInfo()
@@ -81,20 +111,42 @@ while True:
     for i, (ifs, score) in enumerate(best):
         print('%02d) %5.3f %s' % (i, score, ifs))
 
+    # Saving best
+    im = Image.fromarray(best[0][0].get_image())
+    outfile_gen = '/tmp/gen_%03d.png'
+    im.save(outfile_gen % generation)
+
+    im_png = PngImageFile(outfile_gen % generation)
+    metadata = PngInfo()
+    metadata.add_text('Score', '%5.3f' % score)
+    metadata.add_text('Generation', '%04d' % generation)
+    metadata.add_text('Pool_position', '%02d' % 1)
+    for key, text in ifs.expr_dict().items():
+        metadata.add_text(key, text)
+    im_png.save(outfile_gen % generation, pnginfo=metadata)
+
     # Mutating best
     pool = [b for b, _ in best]
     for ifs, _ in best:
-        for _ in range((pool_sz//best_sz)-1):
+        nmut_max = (pool_sz//best_sz)-1
+        for nmut in range(nmut_max):
             # needs >= IFS5
+            again = 0
             while True:
-                new = copy.copy(ifs)
-                new.mutation(n_mutation)
+                # half random mutation and half mutation by adding two IFS
+                if again > 5 or nmut < nmut_max / 2:
+                    new = copy.copy(ifs)
+                    new.mutation(n_mutation)
+                else:
+                    new =  ifs + random.choice(best)[0]
+
                 for cur in pool:
                     if cur - new == 0:
                         break
                 else:
                     pool.append(new)
                     break 
+                again += 1
             ###
             #new = copy.copy(ifs)
             #new.mutation(n_mutation)

python_rpnifs/fractdim.py

@@ -11,10 +11,22 @@ def rgb2gray(rgb):
 
 def rgba2gray(rgba):
     """ @return a grayscale version of an RGBA "image" """
-    r, g, b, a = rgb[:,:,0], rgb[:,:,1], rgb[:,:,2], rgb[:,:,3]
+    r, g, b, a = rgba[:,:,0], rgba[:,:,1], rgba[:,:,2], rgba[:,:,3]
     gray = (0.2989 * r + 0.5870 * g + 0.1140 * b) * (a/255)
     return gray
 
+def rgb2grayavg(rgb):
+    """ @return a grayscale version of an RGB "image" """
+    r, g, b = rgb[:,:,0], rgb[:,:,1], rgb[:,:,2]
+    gray = (r+g+b)/3
+    return gray
+
+def rgba2grayavg(rgba):
+    """ @return a grayscale version of an RGBA "image" """
+    r, g, b, a = rgba[:,:,0], rgba[:,:,1], rgba[:,:,2], rgba[:,:,3]
+    gray = ((r + g + b)/3) * (a/255)
+    return gray
+
 def fractal_dimension(Z, threshold=None):
     ''' @return Minkowski-Bouligand dimension (computed) '''
     # Only for 2d image

python_rpnifs/ifs5.py

@@ -7,14 +7,15 @@ from skimage.restoration import estimate_sigma
 
 from .expr import *
 from .fractdim import *
+from .shape_score import ShapeScore
 
 
 class IFS5(object):
     """ @brief 1 Variable IFS with R,G,B,A X,Y decomposition """
 
     #height=width=1024
-    height=width=768
-    #height=width=512
+    #height=width=768
+    height=width=512
 
     def __init__(self, nexpr=4, init_sz=1, world=None):
         self._nexpr = nexpr
@@ -44,8 +45,6 @@ class IFS5(object):
             for j, el in enumerate(expr):
                 res[k_fmt % (i, el_lbl[j])] = str(el)
         return res
-                
-        
 
     def _fel(self):
         el_lbl = 'XYrgba'
@@ -64,6 +63,9 @@ class IFS5(object):
         #ret._world = copy.deepcopy(self._world)
         return ret
 
+    def get_image(self):
+    	return self._world
+ 
     def __sub__(self, b):
         """ @return The sum of the levensthein distance between all expressions
                 of two ifs
@@ -72,9 +74,23 @@ class IFS5(object):
                          for j in range(len(b._expr[i]))])
                     for i in range(len(b._expr))])
 
-    def get_image(self):
-    	return self._world
- 
+    def __add__(self, b):
+        """ @return A new IFS generated from both IFS, randomly choosing
+            expression from both
+        """
+        if type(b) !=  type(self):
+            raise TypeError('Types are not IFS5 for add')
+
+        ret = IFS5(nexpr=self._nexpr, world=self._world)
+
+        rc = random.choice
+        rint = random.randint
+
+        ret._expr = [[copy.copy(rc([self, b])._expr[rc([i, len(self._expr)-1])][rc([j, len(ex)-1])])
+                      for j, e in enumerate(ex)]
+                     for i, ex in enumerate(self._expr)]
+        return ret
+
     def mutation(self, n=1, rand=True):
         """ @brief Apply mutation on expressions
             @param n the mutation count
@@ -134,7 +150,7 @@ class IFS5(object):
         #alpha score
         #scores += [fractal_dimension(self._world[:,:,3])]
 
-        gray = rgb2gray(self._world)
+        gray = rgba2gray(self._world)
         graysigma = estimate_sigma(gray)
         grayscore = fractal_dimension(gray)
         del(gray)
@@ -144,7 +160,14 @@ class IFS5(object):
 
         scores += [grayscore]*3
 
-        sigma = sum(sigmas)/len(sigmas)
+        sigma_zero = 0
+        for sig in sigmas:
+            if sig == 0:
+                sigma_zero += 1
+        if len(sigmas) > sigma_zero:
+            sigma = sum(sigmas)/(len(sigmas)-sigma_zero)
+        else:
+            sigma = 0
 
         mod = abs(scores[0]-scores[1])
         mod += abs(scores[0]-scores[2])
@@ -155,13 +178,20 @@ class IFS5(object):
 
         null_comp = 0
         for i in range(3):
-            null_comp += 1 if scores[i] == 0 else 0
+            null_comp += 1 if scores[i] <= 0 else 0
 
         if null_comp >= 2:
             score = 0
             mod *= 0.8
         else:
-            score = sum(scores)/len(scores)
+            zero_score=0
+            for score in scores:
+                if score == 0:
+                    zero_score += 1
+            if len(scores) < zero_score:
+                score = sum(scores)/(len(scores)-zero_score)
+            else:
+                score = 0
 
         score += mod 
 

python_rpntoken.c

@@ -0,0 +1,504 @@
+#include "python_rpntoken.h"
+/**@file python_rpntoken.c
+ * @brief Python module & type definition
+ * @ingroup python_ext
+ * @ingroup python_pyrpn_token
+ * @ingroup python_pyrpn_token_Token
+ * @ingroup python_pyrpn_token_TokenOp
+ * @ingroup python_pyrpn_token_TokenVal
+ * @ingroup python_pyrpn_token_TokenArg
+ */
+
+/**@brief @ref pymod_pyrpn_token_Token method definition
+ * @ingroup python_pyrpn_token_Token */
+static PyMethodDef RPNToken_methods[] = {
+	PYRPN_method("from_str", rpntoken_from_str, METH_CLASS | METH_O,
+			"cls, token_str, /",
+			"Return a new RPNToken subclass instance from string"),
+	{NULL} //
+};
+
+PyTypeObject RPNTokenType = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name = "pyrpn.tokens.Token",
+	.tp_doc = "Abstract class for RPN expression tokens",
+	.tp_basicsize = sizeof(RPNToken_t),
+	.tp_itemsize = 0,
+	.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+	.tp_richcompare = rpntoken_richcompare,
+	.tp_init = rpntoken_init,
+	.tp_new = PyType_GenericNew,
+	.tp_str = rpntoken_str,
+	.tp_repr = rpntoken_repr,
+	.tp_methods = RPNToken_methods,
+};
+
+/**@brief @ref pymod_pyrpn_token_TokenOp method definition
+ * @ingroup python_pyrpn_token_TokenOp */
+static PyMethodDef RPNTokenOp_methods[] = {
+	PYRPN_method("opcode_max", rpntokenop_opcode_max,
+			METH_STATIC | METH_NOARGS,
+			"",
+			"Return the maximum valid value for an opcode"),
+	PYRPN_method("chr", rpntokenop_opchr, METH_NOARGS,
+			"self, /",
+			"Return the single char representation of operand"),
+	PYRPN_method("str", rpntokenop_opstr, METH_NOARGS,
+			"self, /",
+			"Return the string (multi-char) representation of the operand"),
+	{NULL} //
+};
+
+/**@brief @ref pymod_pyrpn_token_TokenOp members definition
+ * @ingroup python_pyrpn_token_TokenOp */
+static PyMemberDef RPNTokenOp_members[] = {
+	{"opcode", T_BYTE, offsetof(RPNTokenOp_t, super.value.op_n), READONLY,
+		"The number representing the operand"},
+	/*
+	{"chr", T_CHAR, offsetof(RPNTokenOp_t, super.value.op->chr), READONLY,
+		"The single char representation of the operand"},
+	{"str", T_STRING, offsetof(RPNTokenOp_t, super.value.op->str), READONLY,
+		"The str representation of the operand"},
+	*/
+	{NULL} //
+};
+
+PyTypeObject RPNTokenOpType = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_base = &RPNTokenType,
+	.tp_name = "pyrpn.tokens.Operand",
+	.tp_doc = "RPN expression operand token",
+	.tp_basicsize = sizeof(RPNTokenOp_t),
+	.tp_itemsize = 0,
+	.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+	.tp_init = rpntokenop_init,
+	.tp_members = RPNTokenOp_members,
+	.tp_methods = RPNTokenOp_methods,
+};
+
+/**@brief @ref pymod_pyrpn_token_TokenArg members definition
+ * @ingroup python_pyrpn_token_TokenArg */
+static PyMemberDef RPNTokenArg_members[] = {
+	{"argno", T_ULONG, offsetof(RPNTokenOp_t, super.value.arg_n), READONLY,
+		"The argument number"},
+	{NULL} //
+};
+
+PyTypeObject RPNTokenArgType = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_base = &RPNTokenType,
+	.tp_name = "pyrpn.tokens.Argument",
+	.tp_doc = "RPN expression argument token",
+	.tp_basicsize = sizeof(RPNTokenArg_t),
+	.tp_itemsize = 0,
+	.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, 
+	.tp_init = rpntokenarg_init,
+	.tp_members = RPNTokenArg_members,
+};
+
+/**@brief @ref pymod_pyrpn_token_TokenVal members definition
+ * @ingroup python_pyrpn_token_TokenVal */
+static PyMemberDef RPNTokenVal_members[] = {
+	{"value", T_ULONG, offsetof(RPNTokenOp_t, super.value.value), READONLY,
+		"The immediate value"},
+	{NULL} //
+};
+
+PyTypeObject RPNTokenValType = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_base = &RPNTokenType,
+	.tp_name = "pyrpn.tokens.Value",
+	.tp_doc = "RPN expression value token",
+	.tp_basicsize = sizeof(RPNTokenVal_t),
+	.tp_itemsize = 0,
+	.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, 
+	.tp_init = rpntokenval_init,
+	.tp_members = RPNTokenVal_members,
+};
+
+PyModuleDef rpntokens_module = {
+	PyModuleDef_HEAD_INIT,
+	.m_name = "pyrpn.tokens",
+	.m_doc = "RPN expression tokens classes",
+	.m_size = -1,
+	.m_methods = NULL,
+	.m_clear = NULL,
+};
+
+
+PyObject* rpntokens_module_init(void)
+{
+	PyObject *mod;
+
+	mod = PyModule_Create(&rpntokens_module);
+	if(!mod) { return NULL; }
+
+	if(PyType_Ready(&RPNTokenType) < 0
+			|| PyType_Ready(&RPNTokenOpType) < 0 \
+			|| PyType_Ready(&RPNTokenValType) < 0 \
+			|| PyType_Ready(&RPNTokenArgType) < 0)
+	{
+		return NULL;
+	}
+	Py_INCREF(&RPNTokenType);
+	Py_INCREF(&RPNTokenOpType);
+	Py_INCREF(&RPNTokenValType);
+	Py_INCREF(&RPNTokenArgType);
+
+	if(	PyModule_AddObject(mod, "Token",
+			(PyObject*)&RPNTokenType) < 0 \
+		|| PyModule_AddObject(mod, "Operand",
+			(PyObject*)&RPNTokenOpType) < 0 \
+		|| PyModule_AddObject(mod, "Value",
+			(PyObject*)&RPNTokenValType) < 0 \
+		|| PyModule_AddObject(mod, "Argument",
+			(PyObject*)&RPNTokenArgType) < 0)
+	{
+		Py_DECREF(&RPNTokenType);
+		Py_DECREF(&RPNTokenOpType);
+		Py_DECREF(&RPNTokenValType);
+		Py_DECREF(&RPNTokenArgType);
+		return NULL;
+	}
+
+	return mod;
+}
+
+PyObject* rpntoken_from_str(PyObject *cls, PyObject *arg)
+{
+	if(!PyUnicode_Check(arg))
+	{
+		PyErr_SetString(PyExc_TypeError, "Expected argument to be a str");
+		return NULL;
+	}
+	PyObject *bytes_str = PyUnicode_AsASCIIString(arg);
+	if(PyErr_Occurred())
+	{
+		return NULL;
+	}
+	const char *str = PyBytes_AS_STRING(bytes_str);
+	rpn_token_t token;
+
+	char err_str[64];
+	if(rpn_tokenize(str, &token, err_str) < 0)
+	{
+		Py_DECREF(bytes_str);
+		PyObject *ascii = PyObject_ASCII(arg);
+		PyErr_Format(PyExc_ValueError, "Unable to parse %s", ascii);
+		Py_DECREF(ascii);
+		return NULL;
+	}
+	Py_DECREF(bytes_str);
+
+	return rpntoken_from_token(&token);
+}
+
+
+PyObject* rpntoken_from_token(const rpn_token_t *token)
+{
+	PyTypeObject *type;
+
+	switch(token->type)
+	{
+		case RPN_op:
+			type = &RPNTokenOpType;
+			break;
+		case RPN_val:
+			type = &RPNTokenValType;
+			break;
+		case RPN_arg:
+			type = &RPNTokenArgType;
+			break;
+		default:
+			PyErr_SetString(PyExc_RuntimeError,
+					"Unrecognized parsed token");
+			return NULL;
+	}
+
+	PyObject *ret = PyObject_CallMethod((PyObject*)type, "__new__", "O", type);
+	if(PyErr_Occurred())
+	{
+		return NULL;
+	}
+	// Any child type should work here since struct begins with super
+	RPNToken_t *_ret = (RPNToken_t*)ret;
+	memcpy(&_ret->value, token, sizeof(*token));
+	return ret;
+
+}
+
+
+int rpntoken_init(PyObject *_self, PyObject *args, PyObject *kwds)
+{
+	PyErr_SetString(PyExc_NotImplementedError, "Abstract class");
+	return -1;
+}
+
+PyObject* rpntoken_richcompare(PyObject *_self, PyObject *_other, int op)
+{
+	RPNToken_t *self, *other;
+	self = (RPNToken_t*)_self;
+	other = (RPNToken_t*)_other;
+
+
+	if(!PyObject_IsInstance(_other, (PyObject*)&RPNTokenType))
+	{
+		PyErr_Format(PyExc_TypeError,
+				"Can only be compared with RPNToken subtypes");
+		return NULL;
+	}
+
+	int cmp = self->value.type - other->value.type;
+	if(cmp == 0)
+	{
+		switch(self->value.type)
+		{
+			case RPN_op:
+				cmp = self->value.op_n - other->value.op_n;
+				break;
+			case RPN_arg:
+				cmp = self->value.arg_n - other->value.arg_n;
+				break;
+			case RPN_val:
+				cmp = self->value.value - other->value.value;
+				break;
+			default:
+				PyErr_Format(PyExc_RuntimeError,
+						"Unknown token type %d",
+						self->value.type);
+				return NULL;
+		}
+	}
+
+	switch(op)
+	{
+		case Py_LT:
+			if(cmp < 0) { Py_RETURN_TRUE; }
+			Py_RETURN_FALSE;
+		case Py_LE:
+			if(cmp <= 0) { Py_RETURN_TRUE; }
+			Py_RETURN_FALSE;
+		case Py_EQ:
+			if(cmp == 0) { Py_RETURN_TRUE; }
+			Py_RETURN_FALSE;
+		case Py_NE:
+			if(cmp != 0) { Py_RETURN_TRUE; }
+			Py_RETURN_FALSE;
+		case Py_GT:
+			if(cmp > 0) { Py_RETURN_TRUE; }
+			Py_RETURN_FALSE;
+		case Py_GE:
+			if(cmp >= 0) { Py_RETURN_TRUE; }
+			Py_RETURN_FALSE;
+		default:
+			PyErr_Format(PyExc_NotImplementedError,
+					"Unknown comparison %d",
+					self->value.type);
+			return NULL;
+	}
+}
+
+PyObject* rpntoken_repr(PyObject *_self)
+{
+	RPNToken_t *self = (RPNToken_t*)_self;
+
+
+	int needed_sz = rpn_token_snprintf(&self->value, NULL, 0);
+	if(needed_sz < 0)
+	{
+		PyErr_Format(PyExc_RuntimeError, "Error : %s", strerror(errno));
+		return NULL;
+	}
+	char str[needed_sz+1];
+	rpn_token_snprintf(&self->value, str, needed_sz+1);
+
+	PyTypeObject *tp = Py_TYPE(_self);
+	PyObject *tp_name = PyType_GetName(tp);
+	PyObject *bytes_str = PyUnicode_AsASCIIString(tp_name);
+	Py_DECREF(tp_name);
+	const char *typename = PyBytes_AS_STRING(bytes_str);
+
+	needed_sz = snprintf(NULL, 0, "<%s '%s'>", typename, str);
+	char res_str[needed_sz+1];
+	needed_sz = snprintf(res_str, needed_sz+1, "<%s '%s'>", typename, str);
+	Py_DECREF(bytes_str);
+
+	return PyUnicode_FromString(res_str);
+}
+
+PyObject* rpntoken_str(PyObject *_self)
+{
+	RPNToken_t *self = (RPNToken_t*)_self;
+
+	int needed_sz = rpn_token_snprintf(&self->value, NULL, 0);
+	if(needed_sz < 0)
+	{
+		PyErr_Format(PyExc_RuntimeError, "Error : %s", strerror(errno));
+		return NULL;
+	}
+	char str[needed_sz+1];
+	rpn_token_snprintf(&self->value, str, needed_sz+1);
+	return PyUnicode_FromString(str);
+}
+
+
+int rpntokenop_init(PyObject *_self, PyObject *args, PyObject *kwds)
+{
+	RPNTokenOp_t *self = (RPNTokenOp_t*)_self;
+	PyObject *pyop = NULL;
+	char *names[] = {"op", NULL};
+
+
+	if(!PyArg_ParseTupleAndKeywords(args, kwds, "O:RPNTokenOP.__init__",
+				names,
+				&pyop))
+	{
+		return -1;
+	}
+
+	Py_INCREF(pyop);
+
+	if(PyLong_Check(pyop))
+	{
+		//opcode given ?
+		long opcode = PyLong_AsLong(pyop);
+		if(PyErr_Occurred()) { return -1; }
+		if(opcode < 0)
+		{
+			PyErr_SetString(PyExc_ValueError, "Opcode cannot be negative");
+			return -1;
+		}
+		else if ((size_t)opcode >= rpn_op_sz())
+		{
+			PyErr_Format(PyExc_ValueError,
+					"Maximum opcode is %ld but %ld given",
+					rpn_op_sz()-1, opcode);
+			return -1;
+		}
+		self->super.value.op_n = opcode;
+		self->super.value.op = rpn_op_from_opcode(opcode);
+	}
+	else if(PyUnicode_Check(pyop))
+	{
+		PyObject *bytes_str = PyUnicode_AsASCIIString(pyop);
+		if(PyErr_Occurred())
+		{
+			return -1;
+		}
+		const char *token_str = PyBytes_AS_STRING(bytes_str);
+		char err_str[64];
+
+		if(rpn_tokenize(token_str, &(self->super.value), err_str) < 0)
+		{
+			Py_DECREF(bytes_str);
+			PyObject *ascii = PyObject_ASCII(pyop);
+			PyErr_Format(PyExc_ValueError, "Unrecognized token '%s' : %s",
+					ascii, err_str);
+			Py_DECREF(ascii);
+			goto err;
+		}
+		Py_DECREF(bytes_str);
+		if(self->super.value.type != RPN_op)
+		{
+			PyErr_SetString(PyExc_TypeError, "Decoded token is not an operand");
+			goto err;
+		}
+	}
+	else
+	{
+		PyErr_SetString(PyExc_TypeError, "Given argument is neither str neither int");
+		goto err;
+
+	}
+
+	Py_DECREF(pyop);
+	return 0;
+
+err:
+	if(pyop)
+	{
+		Py_DECREF(pyop);
+	}
+	return -1;
+}
+
+PyObject *rpntokenop_opcode_max(PyObject* Py_UNUSED(_static))
+{
+	return PyLong_FromSize_t(rpn_op_sz()-1);
+}
+
+PyObject *rpntokenop_opchr(PyObject *_self, PyObject*  Py_UNUSED(_null))
+{
+	RPNTokenOp_t *self = (RPNTokenOp_t*)_self;
+	if(self->super.value.op->chr == '\0')
+	{
+		Py_RETURN_NONE;
+	}
+	char buf[2];
+	buf[1] = '\0';
+	buf[0] = self->super.value.op->chr;
+	return PyUnicode_FromString(buf);
+}
+
+
+PyObject *rpntokenop_opstr(PyObject *_self, PyObject*  Py_UNUSED(_null))
+{
+	RPNTokenOp_t *self = (RPNTokenOp_t*)_self;
+	if(!self->super.value.op->str)
+	{
+		Py_RETURN_NONE;	
+	}
+	Py_RETURN_NONE;	
+	return PyUnicode_FromString(self->super.value.op->str);
+}
+
+int rpntokenval_init(PyObject *_self, PyObject *args, PyObject *kwds)
+{
+	RPNTokenOp_t *self = (RPNTokenOp_t*)_self;
+	char *names[] = {"value", NULL};
+	PyObject *arg = NULL;
+
+	if(!PyArg_ParseTupleAndKeywords(args, kwds, "O:RPNTokenVal.__init__",
+				names, &arg))
+	{
+		return -1;
+	}
+	if(!PyLong_Check(arg))
+	{
+		PyErr_SetString(PyExc_TypeError, "Expected integer as argument");
+		return -1;
+	}
+	self->super.value.value = PyLong_AsUnsignedLong(arg);
+	if(PyErr_Occurred())
+	{
+		return -1;
+	}
+	self->super.value.type = RPN_val;
+	return 0;
+}
+
+int rpntokenarg_init(PyObject *_self, PyObject *args, PyObject *kwds)
+{
+	RPNTokenOp_t *self = (RPNTokenOp_t*)_self;
+	char *names[] = {"argno", NULL};
+	PyObject *arg = NULL;
+
+	if(!PyArg_ParseTupleAndKeywords(args, kwds, "O:RPNTokenArg.__init__",
+				names, &arg))
+	{
+		return -1;
+	}
+	if(!PyLong_Check(arg))
+	{
+		PyErr_SetString(PyExc_TypeError, "Expected integer as argument");
+		return -1;
+	}
+	self->super.value.arg_n = PyLong_AsUnsignedLong(arg);
+	if(PyErr_Occurred())
+	{
+		return -1;
+	}
+	self->super.value.type = RPN_arg;
+	return 0;
+}
+

python_rpntoken.h

@@ -0,0 +1,211 @@
+/*
+ * Copyright (C) 2023 Weber Yann
+ * 
+ * This file is part of pyrpn.
+ * 
+ * pyrpn is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * 
+ * pyrpn is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with pyrpn.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _PYTHON_RPNTOKEN_H__
+#define _PYTHON_RPNTOKEN_H__
+
+#include "config.h"
+#include "rpn_parse.h"
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include "structmember.h"
+
+/**@file python_rpntoken.h
+ * @brief Python RPNToken type headers
+ * @ingroup python_ext
+ * @ingroup pymod_pyrpn_token
+ *
+ * This file is the header of the RPNToken classes and subclasses
+ *
+ */
+
+/**@defgroup pymod_pyrpn_token module pyrpn.token
+ * @brief Python module representing RPNExpr tokens
+ * @ingroup pymod_pyrpn
+ */
+/**@brief pyrpn.token module */
+extern PyModuleDef rpntokens_module;
+
+/**@defgroup pymod_pyrpn_token_Token pyrpn.token.Token
+ * @brief Abstract class representing an @ref pymod_pyrpn_RPNExpr token
+ * @ingroup pymod_pyrpn_token */
+/**@brief pyrpn.token.Token generic type
+ * @ingroup pymod_pyrpn_token_Token */
+extern PyTypeObject RPNTokenType;
+
+/**@defgroup pymod_pyrpn_token_TokenOp pyrpn.token.TokenOp
+ * @brief Python class representing an @ref pymod_pyrpn_RPNExpr operation token
+ *
+ * Extends @ref pymod_pyrpn_token_Token
+ * @ingroup pymod_pyrpn_token */
+/**@brief pyrpn.token.TokenOp type
+ * @ingroup pymod_pyrpn_token_TokenOp */
+extern PyTypeObject RPNTokenOpType;
+
+/**@defgroup pymod_pyrpn_token_TokenVal pyrpn.token.TokenVal
+ * @brief Python class representing an @ref pymod_pyrpn_RPNExpr value token
+ *
+ * Extends @ref pymod_pyrpn_token_Token
+ * @ingroup pymod_pyrpn_token */
+/**@brief pyrpn.token.TokenVal type
+ * @ingroup pymod_pyrpn_token_TokenVal */
+extern PyTypeObject RPNTokenValType;
+
+/**@defgroup pymod_pyrpn_token_TokenArg pyrpn.token.TokenArg
+ * @brief Python class representing an @ref pymod_pyrpn_RPNExpr argument token
+ *
+ * Extends @ref pymod_pyrpn_token_Token
+ * @ingroup pymod_pyrpn_token */
+/**@brief pyrpn.token.TokenArg type
+ * @ingroup pymod_pyrpn_token_TokenArg */
+extern PyTypeObject RPNTokenArgType;
+
+
+/**@brief C representation of @ref RPNTokenType generic class
+ * @ingroup pymod_pyrpn_token_Token */
+typedef struct
+{
+	/** Python's type definition */
+	PyObject_HEAD;
+	/** @brief The token value (will be set by subtypes) */
+	rpn_token_t value;
+} RPNToken_t;
+
+/**@brief C representation of all @ref RPNTokenType subclasses
+ * @ingroup pymod_pyrpn_token_tokenOp
+ * @ingroup pymod_pyrpn_token_tokenVal
+ * @ingroup pymod_pyrpn_token_tokenArg
+ */
+typedef struct {
+	/** @brief Pointer on super type */
+	RPNToken_t super;
+} RPNTokenSubClass_t;
+
+/**@brief C representation of @ref RPNTokenOpType
+ * @ingroup pymod_pyrpn_token_TokenOp */
+typedef RPNTokenSubClass_t RPNTokenOp_t;
+/**@brief C representation of @ref RPNTokenValType
+ * @ingroup pymod_pyrpn_token_TokenVal */
+typedef RPNTokenSubClass_t RPNTokenVal_t;
+/**@brief C representation of @ref RPNTokenArgType
+ * @ingroup pymod_pyrpn_token_TokenArg */
+typedef RPNTokenSubClass_t RPNTokenArg_t;
+
+
+/**@brief Module initialisation function 
+ * @return initialized module
+ * @ingroup pymod_pyrpn_token */
+PyObject* rpntokens_module_init(void);
+
+/**@brief Class method returning a token instance from a string representation
+ * @param cls @ref RPNTokenType class or any subclass
+ * @param arg A str representing a token
+ * @return A new @ref RPNTokenType subclass instance or NULL on error
+ * @ingroup pymod_pyrpn_token_Token
+ */
+PyObject* rpntoken_from_str(PyObject *cls, PyObject *arg);
+
+/**@brief Instanciate a new RPNToken subclass given a C token
+ * @param token An expression token
+ * @return A new @ref RPNTokenType subclass instance
+ * @ingroup pymod_pyrpn_token_Token
+ */
+PyObject* rpntoken_from_token(const rpn_token_t *token);
+
+
+/**@brief @ref RPNTokenType __init__ method
+ * @param _self @ref RPNTokenType subclass instance
+ * @param args Positional arguments
+ * @param kwds Keyword arguments
+ * @return 0 or -1 on error
+ * @ingroup pymod_pyrpn_token_Token
+ */
+int rpntoken_init(PyObject *_self, PyObject *args, PyObject *kwds);
+
+/**@brief PEP-207 comparison method
+ * @param _self An @ref RPNTokenType subclass instance
+ * @param other The @ref RPNTokenType subclass instance to compare to
+ * @param op The comparison
+ * @return Py_True or Py_False
+ * @ingroup pymod_pyrpn_token_Token
+ */
+PyObject* rpntoken_richcompare(PyObject *_self, PyObject *other, int op);
+
+/**@brief __str__ method
+ * @param _self An @ref RPNTokenType subclass instance
+ * @return A str representing the token
+ * @ingroup pymod_pyrpn_token_Token */
+PyObject* rpntoken_str(PyObject *_self);
+
+/**@brief __repr__ method
+ * @param _self An @ref RPNTokenType subclass instance
+ * @return A str representing the token
+ * @ingroup pymod_pyrpn_token_Token */
+PyObject* rpntoken_repr(PyObject *_self);
+
+
+
+
+
+/**@brief @ref RPNTokenOpType __init__ method
+ * @param _self RPNTokenOpType being initialized
+ * @param args Positional arguments
+ * @param kwds Keyword arguments
+ * @return 0 or -1 on error
+ * @ingroup pymod_pyrpn_token_TokenOp */
+int rpntokenop_init(PyObject *_self, PyObject *args, PyObject *kwds);
+
+/**@brief @ref RPNTokenOpType opcode_max method
+ * @param Py_UNUSED unused argument for static method
+ * @return The maximum valid opcode
+ * @ingroup pymod_pyrpn_token_TokenOp */
+PyObject *rpntokenop_opcode_max(PyObject *Py_UNUSED(_static));
+
+/**@brief @ref RPNTokenOpType chr method
+ * @param _self @ref RPNTokenOpType instance
+ * @param Py_UNUSED unused argument
+ * @return A string with a single chr representing the operation
+ * @ingroup pymod_pyrpn_token_TokenOp */
+PyObject *rpntokenop_opchr(PyObject *_self, PyObject*  Py_UNUSED(_null));
+
+/**@brief @ref RPNTokenOpType str method
+ * @param _self @ref RPNTokenOpType instance
+ * @param Py_UNUSED unused argument
+ * @return A string representing the operation on multiple chr
+ * @ingroup pymod_pyrpn_token_TokenOp */
+PyObject *rpntokenop_opstr(PyObject *_self, PyObject*  Py_UNUSED(_null));
+
+
+/**@brief @ref RPNTokenValType __init__ method
+ * @param _self RPNTokenValType being initialized
+ * @param args Positional arguments
+ * @param kwds Keyword arguments
+ * @return 0 or -1 on error
+ * @ingroup pymod_pyrpn_token_TokenVal */
+int rpntokenval_init(PyObject *_self, PyObject *args, PyObject *kwds);
+
+/**@brief @ref RPNTokenArgType __init__ method
+ * @param _self RPNTokenArgType being initialized
+ * @param args Positional arguments
+ * @param kwds Keyword arguments
+ * @return 0 or -1 on error
+ * @ingroup pymod_pyrpn_token_TokenArg  */
+int rpntokenarg_init(PyObject *_self, PyObject *args, PyObject *kwds);
+
+#endif

rpn_if.c

@@ -18,7 +18,8 @@
  */
 #include "rpn_if.h"
 
-rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap)
+rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap,
+		rpn_expr_t *init_exprs)
 {
 	rpn_if_t *res;
 	size_t i;
@@ -46,7 +47,8 @@ rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap)
 	else
 	{
 		res->self_mem = 1;
-		res->mem = mmap(NULL, params->mem_sz, PROT_READ|PROT_WRITE,
+		res->mem = mmap(NULL, params->mem_sz * params->value_sz,
+			PROT_READ|PROT_WRITE,
 			MAP_ANON | MAP_PRIVATE, -1, 0);
 		if(res->mem == (void*)-1)
 		{
@@ -70,9 +72,40 @@ rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap)
 		#endif
 		goto rpn_malloc_err;
 	}
+
 	res->rpn_args = &(res->rpn_res[params->rpn_sz]);
 
-	res->rpn = malloc(sizeof(rpn_expr_t*) * params->rpn_sz);
+	if(init_exprs)
+	{ // using existing exppressions, checking them
+		short err = 0;
+		for(size_t i=0; i<params->rpn_sz; i++)
+		{
+			if(init_exprs[i].args_count != params->rpn_argc)
+			{
+				err = 1;
+				snprintf(init_exprs[i].err_reason, 128,
+						"Expected %ld arguments but expression got %ld",
+						params->rpn_argc,
+						init_exprs[i].args_count);
+			}
+			else if(init_exprs[i].stack_sz != params->rpn_stack_sz)
+			{
+				err = 1;
+				snprintf(init_exprs[i].err_reason, 128,
+						"Expected %d element stack but expression got %d",
+						params->rpn_stack_sz,
+						init_exprs[i].stack_sz);
+			}
+		}
+		if(err)
+		{
+			goto rpn_malloc_err;
+		}
+		res->rpn = init_exprs;
+		return res;
+	}
+
+	res->rpn = malloc(sizeof(rpn_expr_t) * params->rpn_sz);
 	if(!res->rpn)
 	{
 		#if DEBUG
@@ -94,6 +127,7 @@ rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap)
 			#endif
 			goto rpn_init_error;
 		}
+		rpn_expr_compile(&(res->rpn[i]), "");
 	}
 
 	return res;
@@ -112,7 +146,7 @@ rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap)
 		err = errno;
 		if(res->self_mem)
 		{
-			munmap(res->mem, params->mem_sz);
+			munmap(res->mem, params->mem_sz * params->value_sz);
 		}
 	mmap_err:
 		err = errno;
@@ -134,26 +168,26 @@ void rpn_if_free(rpn_if_t* rif)
 	free(rif->rpn_res);
 	if(rif->self_mem)
 	{
-		munmap(rif->mem, rif->params->mem_sz);
+		munmap(rif->mem, rif->params->mem_sz * rif->params->value_sz);
 	}
 	free(rif);
 }
 
 size_t rpn_if_step(rpn_if_t *rif, size_t pos)
 {
-	size_t i;
 	size_t newpos;
-	rif->params->arg_f(rif, pos, rif->rpn_args);
-	/* WRONG ! rif->rpn_args is in rif structure and do not have to be
-	   given as argument... */
-	for(i=0; i<rif->params->rpn_sz; i++)
+
+	assert(rif != NULL);
+	assert(rif->params != NULL);
+	assert(rif->params->getarg_f != NULL);
+
+	rif->params->getarg_f(rif, pos);
+	for(size_t i=0; i<rif->params->rpn_sz; i++)
 	{
-		rif->rpn_res[i] = rpn_expr_eval(&(rif->rpn[i]), rif->rpn_args);
+		rpn_value_t res = rpn_expr_eval(&(rif->rpn[i]), rif->rpn_args);
+		rif->rpn_res[i] = res;
 	}
-	//rif->params->res_f(rif, &newpos, rif->rpn_res);
-	/* WRONG ! rif->rpn_res is in rif structure and do not have to be
-	   given as argument... */
-	rif->params->res_f(rif, &newpos, NULL);
+	rif->params->setres_f(rif, &newpos);
 	return newpos;
 }
 

rpn_if.h

@@ -21,6 +21,8 @@
 
 #include "config.h"
 
+#include <assert.h>
+
 #include "rpn_jit.h"
 
 /**@file rpn_if.h
@@ -34,12 +36,29 @@
  * @ingroup ifs
  * @brief Iterated RPN expression
  *
- * A single Iterated Function implemented using an RPN expression.
+ * @note For more details about Iterated function see dedicated
+ * section : @ref doc_ifs_if
  *
- * @note The goal is to optimize iteration writing the code in C and providing
- * an high level Python API.
+ * Iterated function are function that can be iterated on successive position
+ * in a memory map. The function takes its arguments from a position in the
+ * memory map and output its results at a resulting position in the memory map.
  *
- * For more details about IF see dedicated page : @ref doc_ifs_if
+ * The iterated function can be seen as a function taking a position as argument
+ * and outputing another position with borders effects from data stored at given
+ * positions.
+ *
+ * The implementation try to be as generic as possible, allowing to implement
+ * the transformation using a @ref rpn_expr_t system. The iterated function
+ * has 2 functions as parameters :
+ * - The first one @ref rpn_if_param_s.getarg_f transform a position
+ *   to argument(s) and fetch the other arguments from the memory map
+ * - The second @ref rpn_if_param_s.setres_f takes the results from the
+ *   @ref rpn_expr_t system, transform it in position and results and set
+ *   the memory map at resulting position with results.
+ *
+ * Based on this generic implementation @ref ifs_if_default are implemented for
+ * various common cases : mutli-dimentionnal positions and constants, boolean,
+ * colors as results.
  */
 
 /**@brief Shortcut for struct @ref rpn_if_s */
@@ -52,7 +71,8 @@ struct rpn_if_param_s
 {
 	/**@brief Memory map size in items */
 	size_t mem_sz;
-	/**@brief Size of a memory item */
+	/**@brief Size of a memory item in bytes (should be a multiple of
+	   sizeof(rpn_value_t))*/
 	size_t value_sz;
 	/**@brief RPN expression count */
 	size_t rpn_sz;
@@ -65,11 +85,10 @@ struct rpn_if_param_s
 	/**@brief Set RPN arguments given a position
 	 * @note transform position to arguments and fetch other arguments
 	 * from memory map*/
-	int (*arg_f)(rpn_if_t *rif, size_t pos, rpn_value_t *args);
+	int (*getarg_f)(rpn_if_t *rif, size_t pos);
 	/**@brief RPN results to data and position transformation
 	 * @note set memory maps with converted data */
-	int (*res_f)(rpn_if_t *rif, size_t *pos,
-		rpn_value_t *data);
+	int (*setres_f)(rpn_if_t *rif, size_t *pos);
 
 	/**@brief Arbitrary data, if set must be freed in one free() call */
 	void *data;
@@ -96,15 +115,17 @@ struct rpn_if_s
 /**@brief Macro fetching a memory pointer given a position
  * @return rpn_value_t* values
  */
-#define rpn_if_getitem(rif, pos) (rif->mem + ((rif->params->value_sz) * pos))
+#define rpn_if_getitem(rif, pos) (rpn_value_t*)(((unsigned char*)rif->mem + ((rif->params->value_sz) * pos)))
 
 /**@brief Alloc a new @ref rpn_if_s using given parameters
  * @param params IF parameters
- * @param rpn list of RPN expresions of params->rpn_sz size
  * @param memmap A suitable memory map. If NULL given, a new mmap is used
+ * @param init_exprs If no NULL uses this expression (steal refs), else uses
+ * 	new empty expressions
  * @return A pointer on an allocated @ref rpn_if_s
  */
-rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap);
+rpn_if_t* rpn_if_new(const rpn_if_param_t *params, rpn_value_t *memmap,
+		rpn_expr_t *init_exprs);
 
 /**@brief Deallocate an @ref rpn_if_s and its ressources and close associated 
  * @ref rpn_expr_s
@@ -121,11 +142,13 @@ size_t rpn_if_step(rpn_if_t *rif, size_t pos);
 
 
 /**@brief Returns the list of RPN expression : allowing to recompile them
+ * @param rif Pointer on IF
  * @return A list of RPN expressions
  * @note The memory area returned must not be freed !
  */
 rpn_expr_t **rpn_if_rpn_get(rpn_if_t *rif);
 
+
 /**@brief New @ref rpn_if_s and partial initialisation
  * @param mem_sz memory size in bytes
  * @param rpn_argc number of arguments taken by @ref rpn_expr_s

rpn_if_default.c

@@ -1,23 +1,27 @@
 #include "rpn_if_default.h"
 
 rpn_if_param_t* rpn_if_default_params(short pos_flag, short res_flag,
-	const size_t *lim, const rpn_value_t *val, unsigned char rpn_stack_sz)
+	const size_t *lim, const rpn_value_t *res_const,
+	unsigned char rpn_stack_sz)
 {
 	rpn_if_param_t *res;
 	rpn_if_default_data_t *data;
-	size_t lim_sz, const_val_sz, param_sz, rpn_sz, mem_sz, argc, i;
+	size_t lim_sz, const_val_sz, param_sz, rpn_sz, mem_sz, ndim, value_sz, argc, i;
 
 	// Calculating full params + default_data + size_lim + const_val size
+	short lim_off = 0;
 	switch(pos_flag)
 	{
 		case RPN_IF_POSITION_LINEAR:
-			lim_sz = 1;
+			ndim = lim_sz = 1;
 			break;
 		case RPN_IF_POSITION_XY:
-			lim_sz = 2;
+			ndim = lim_sz = 2;
 			break;
 		case RPN_IF_POSITION_XDIM:
-			lim_sz = *lim;
+			ndim = lim_sz = *lim;
+			lim_sz++;
+			lim_off = 1;
 			break;
 		default:
 			fprintf(stderr,
@@ -26,11 +30,11 @@ rpn_if_param_t* rpn_if_default_params(short pos_flag, short res_flag,
 			return NULL;
 	}
 	mem_sz = 1;
-	for(i=1;i<lim_sz;i++)
+	for(i=0;i<ndim;i++)
 	{
-		mem_sz *= lim[i];
+		mem_sz *= lim[i+lim_off];
 	}
-	argc = rpn_sz = lim_sz;
+	argc = rpn_sz = ndim;
 	lim_sz *= sizeof(size_t);
 
 	const_val_sz = 0;
@@ -39,29 +43,42 @@ rpn_if_param_t* rpn_if_default_params(short pos_flag, short res_flag,
 		case RPN_IF_RES_BOOL:
 			rpn_sz += 1;
 			argc += 1;
+			value_sz = sizeof(rpn_value_t);
 			break;
 		case RPN_IF_RES_COUNT:
 			argc += 1;
+			value_sz = sizeof(rpn_value_t);
 			break;
 		case RPN_IF_RES_XFUN:
 			rpn_sz += 1;
 			argc += 1;
+			value_sz = sizeof(rpn_value_t);
 			break;
 		case RPN_IF_RES_RGB:
 			rpn_sz += 3;
 			argc += 3;
+			value_sz = sizeof(rpn_value_t) * 3;
 			break;
 		case RPN_IF_RES_RGBA:
 			rpn_sz += 4;
 			argc += 4;
+			value_sz = sizeof(rpn_value_t) * 4;
 			break;
 		case RPN_IF_RES_CONST:
 			const_val_sz = 1;
 			argc += 1;
+			value_sz = sizeof(rpn_value_t);
 			break;
+		/*
+		case RPN_IF_RES_CONST_RGB:
+			const_val_sz = 3;
+			argc += 3;
+			break;
+		*/
 		case RPN_IF_RES_CONST_RGBA:
 			const_val_sz = 4;
 			argc += 4;
+			value_sz = sizeof(rpn_value_t) * 4;
 			break;
 		default:
 			fprintf(stderr,
@@ -69,13 +86,13 @@ rpn_if_param_t* rpn_if_default_params(short pos_flag, short res_flag,
 				pos_flag);
 			return NULL;
 	}
-	if(const_val_sz && !val)
+	if(const_val_sz && !res_const)
 	{
 		fprintf(stderr,
 			"Missing values when creating if params");
 		return NULL;
 	}
-	else if(!const_val_sz && val)
+	else if(!const_val_sz && res_const)
 	{
 		//Warning
 	}
@@ -91,40 +108,93 @@ rpn_if_param_t* rpn_if_default_params(short pos_flag, short res_flag,
 		perror("Unable to alloc iterated function params");
 		return NULL;
 	}
-	res->data = data = (rpn_if_default_data_t*)(&(res[1]));
+	data = (rpn_if_default_data_t*)(res + 1);
+	bzero(res, sizeof(*res));
+	bzero(data, sizeof(*data));
+	res->data = data;
 
-	data->size_lim = (size_t*)&(data[1]);
 	data->pos_flag = pos_flag;
 	data->res_flag = res_flag;
+	data->ndim = ndim;
+	data->size_lim = (size_t*)&(data[1]);
+	data->const_val = ((void*)data->size_lim) + lim_sz;
 
 	memcpy(data->size_lim, lim, lim_sz);
+
 	if(const_val_sz)
 	{
-		data->const_val = (rpn_value_t*)(&(data->size_lim[1]));
-		memcpy(data->const_val, val, const_val_sz);
+		memcpy(data->const_val, res_const, const_val_sz);
 	}
 	else
 	{
 		data->const_val = NULL;
 	}
 
-	res->arg_f = rpn_if_argf_default;
-	res->res_f = rpn_if_resf_default;
+	res->getarg_f = rpn_if_getarg_default;
+	res->setres_f = rpn_if_setres_default;
 
 	res->rpn_argc = argc;
 	res->rpn_stack_sz = rpn_stack_sz;
-	res->value_sz = 1; /* @TODO set res->value_sz with a good value.. */
-	res->mem_sz = mem_sz * res->value_sz;
+	res->value_sz = value_sz;
+	res->mem_sz = mem_sz;
 	res->rpn_sz = rpn_sz;
 
 	return res;
 }
 
-int rpn_if_argf_default(rpn_if_t *rif, size_t pos, rpn_value_t *args)
+int rpn_if_sizes_from_flag(short pos_flag, short res_flag, short sizes[2])
+{
+	short *lim_sz = &sizes[0];
+	short *const_sz = &sizes[1];
+
+	*lim_sz = *const_sz = -1;
+
+	switch(pos_flag)
+	{
+		case RPN_IF_POSITION_LINEAR:
+			*lim_sz = 1;
+			break;
+		case RPN_IF_POSITION_XY:
+			*lim_sz = 2;
+			break;
+		case RPN_IF_POSITION_XDIM:
+			*lim_sz = 1;
+			break;
+		default:
+			return -1;
+	}
+	switch(res_flag)
+	{
+		case RPN_IF_RES_CONST:
+			*const_sz = 1;
+			break;
+		/*
+		case RPN_IF_RES_CONST_RGB:
+			*const_sz = 3;
+			break;
+		*/
+		case RPN_IF_RES_CONST_RGBA:
+			*const_sz = 4;
+			break;
+		case RPN_IF_RES_BOOL:
+		case RPN_IF_RES_COUNT:
+		case RPN_IF_RES_XFUN:
+		case RPN_IF_RES_RGB:
+		case RPN_IF_RES_RGBA:
+			*const_sz = 0;
+			break;
+		default:
+			return -1;
+	}
+	return 0;
+}
+
+int rpn_if_getarg_default(rpn_if_t *rif, size_t pos)
 {
 	size_t cur_arg, i, rgb_imax;
 	rpn_if_default_data_t *data;
 	rpn_value_t *values;
+	rpn_value_t *args = rif->rpn_args;
 
 	data = (rpn_if_default_data_t*)rif->params->data;
 
@@ -142,6 +212,8 @@ int rpn_if_argf_default(rpn_if_t *rif, size_t pos, rpn_value_t *args)
 			rpn_if_argf_xdim(rif, pos, args);
 			cur_arg = *(data->size_lim);
 			break;
+		default:
+			return -1;
 	}
 	if(cur_arg > rif->params->rpn_argc)
 	{
@@ -181,7 +253,7 @@ int rpn_if_argf_default(rpn_if_t *rif, size_t pos, rpn_value_t *args)
 	return -1;
 }
 
-int rpn_if_resf_default(rpn_if_t *rif, size_t *pos, rpn_value_t *res)
+int rpn_if_setres_default(rpn_if_t *rif, size_t *pos)
 {
 	rpn_if_default_data_t *data;
 	size_t cur_arg, i, rgb_imax;
@@ -192,38 +264,42 @@ int rpn_if_resf_default(rpn_if_t *rif, size_t *pos, rpn_value_t *res)
 	switch(data->pos_flag)
 	{
 		case RPN_IF_POSITION_LINEAR:
-			rpn_if_resf_linear(rif, pos, res);
+			rpn_if_resf_linear(rif, pos);
 			cur_arg = 1;
 			break;
 		case RPN_IF_POSITION_XY:
-			rpn_if_resf_xy(rif, pos, res);
+			rpn_if_resf_xy(rif, pos);
 			cur_arg = 2;
 			break;
 		case RPN_IF_POSITION_XDIM:
-			rpn_if_resf_xdim(rif, pos, res);
+			rpn_if_resf_xdim(rif, pos);
 			cur_arg = *(data->size_lim);
 			break;
+		default:
+			return -1;
 	}
 	if(cur_arg > rif->params->rpn_argc)
 	{
 		/** LOG ERROR ! should never append... */
 		return -1;
 	}
-	rgb_imax = 3; /* rgba */
+	rgb_imax = 3; /* rgb */
 	values = rpn_if_getitem(rif, *pos);
-	/**@todo if(res) set the values in res too ! */
 	switch(data->res_flag)
 	{
 		case RPN_IF_RES_BOOL:
 			*values = 1;
+			*rif->rpn_res = *values;
 			break;
 
 		case RPN_IF_RES_CONST:
 			*values = *(data->const_val);
+			*rif->rpn_res = *values;
 			break;
 
 		case RPN_IF_RES_COUNT:
 			(*values)++;
+			*rif->rpn_res = *values;
 			break;
 
 		case RPN_IF_RES_CONST_RGBA:
@@ -232,6 +308,7 @@ int rpn_if_resf_default(rpn_if_t *rif, size_t *pos, rpn_value_t *res)
 			for(i=0;i<rgb_imax;i++)
 			{
 				values[i] = data->const_val[i];
+				rif->rpn_res[i] = values[i];
 			}
 			break;
 
@@ -274,7 +351,7 @@ int rpn_if_argf_linear(rpn_if_t *rif, size_t pos, rpn_value_t *args)
 	return 0;
 }
 
-int rpn_if_resf_linear(rpn_if_t *rif, size_t *pos, rpn_value_t *_data)
+int rpn_if_resf_linear(rpn_if_t *rif, size_t *pos)
 {
 	rpn_if_default_data_t *data;
 	size_t res;
@@ -314,7 +391,7 @@ int rpn_if_argf_xy(rpn_if_t *rif, size_t pos, rpn_value_t *args)
 	return 0;
 }
 
-int rpn_if_resf_xy(rpn_if_t *rif, size_t *pos, rpn_value_t *_data)
+int rpn_if_resf_xy(rpn_if_t *rif, size_t *pos)
 {
 	rpn_if_default_data_t *data;
 	size_t xy[2];
@@ -357,13 +434,12 @@ int rpn_if_argf_xdim(rpn_if_t *rif, size_t pos, rpn_value_t *args)
 	{
 		return -1;
 	}
-	/**@todo check if *(data->size_lim) overflow rif->params->rpn_argc */
 
 	curdim_sz = 1;
 	curpos = pos;
 	for(i=0; i<*(data->size_lim)-1; i++)
 	{
-		curdim_sz *= data->size_lim[i+1];
+		curdim_sz = data->size_lim[i+1];
 		args[i] = curpos % curdim_sz;
 		curpos /= curdim_sz;
 	}
@@ -379,10 +455,10 @@ int rpn_if_argf_xdim(rpn_if_t *rif, size_t pos, rpn_value_t *args)
 	return 0;
 }
 
-int rpn_if_resf_xdim(rpn_if_t *rif, size_t *pos, rpn_value_t *_data)
+int rpn_if_resf_xdim(rpn_if_t *rif, size_t *pos)
 {
 	rpn_if_default_data_t *data;
-	size_t i, res, cur, curlim, prevlim;
+	size_t i, res, cur, curlim, dim_sz;
 
 	data = (rpn_if_default_data_t*)rif->params->data;
 	res = 0;
@@ -395,22 +471,11 @@ int rpn_if_resf_xdim(rpn_if_t *rif, size_t *pos, rpn_value_t *_data)
 	{
 		return -1;
 	}
-	/**@todo check if *(data->size_lim) overflow rif->params->rpn_argc */
 
-	res = rif->rpn_res[0];
-	if(res >= data->size_lim[1])
-	{
-		if(data->pos_flag & RPN_IF_POSITION_OF_ERR)
-		{
-			return -1;
-		}
-		res %= data->size_lim[1];
-	}
-
-	for(i=1; i < *(data->size_lim); i++)
+	dim_sz = 1;
+	for(i=0;  i < *(data->size_lim); i++)
 	{
 		cur = rif->rpn_res[i];
-		prevlim = data->size_lim[i];
 		curlim = data->size_lim[i+1];
 		if(cur >= curlim)
 		{
@@ -420,7 +485,8 @@ int rpn_if_resf_xdim(rpn_if_t *rif, size_t *pos, rpn_value_t *_data)
 			}
 			cur %= curlim;
 		}
-		res += cur * prevlim;
+		res += cur * dim_sz;
+		dim_sz *= curlim;
 	}
 	*pos = res;
 	return 0;

rpn_if_default.h

@@ -18,6 +18,8 @@
  */
 #ifndef __rpn_if_default__h__
 #define __rpn_if_default__h__
+#include "config.h"
+#include "rpn_if.h"
 
 /**@file rpn_if_default.h Defines default IF
  * @ingroup ifs_if_default
@@ -25,17 +27,19 @@
  * @brief Default IF definitions
  */
 
-/**@defgroup ifs_if_default Default functions
+/**@defgroup ifs_if_default Default iterated functions
  * @ingroup ifs_if
- * @brief Simple iterated functions functions
+ * @brief Iterated function default implementation
  *
- * Defines default @ref rpn_if_param_s.res_f and @ref rpn_if_param_s.arg_f
- * functions.
+ * Defines default @ref rpn_if_param_s.setres_f and @ref rpn_if_param_s.getarg_f
+ * functions and flags to select them (see @ref ifs_if_default_posflag and 
+ * @ref ifs_if_default_resflag ).
+ *
+ * The @ref rpn_if_default_params function constructs suitable
+ * @ref rpn_if_param_t to instanciate an @ref rpn_if_t with
+ * @ref rpn_if_new function.
  */
 
-#include "config.h"
-#include "rpn_if.h"
-
 /**@weakgroup ifs_if_default_posflag Default IF position flags
  * @ingroup ifs_if_default
  * @{ */
@@ -65,6 +69,7 @@ typedef struct rpn_if_default_data_s rpn_if_default_data_t;
 /**@brief Stores default IF data
  *
  * Stores flags and size limit
+ * @ingroup ifs_if_default
  */
 struct rpn_if_default_data_s
 {
@@ -77,11 +82,15 @@ struct rpn_if_default_data_s
 	 * @note If NULL no limit
 	 * - For @ref RPN_IF_POSITION_LINEAR size_lim is a single size_t
 	 * - For @ref RPN_IF_POSITION_XY size_lim is two size_t (height and width)
-	 * - For @ref RPN_IF_POSITION_XDIM *size_lim is the size of size_lim
+	 * - For @ref RPN_IF_POSITION_XDIM the first element (*size_lim) is the
+	 *   size of the size_lim array
 	 */
 	size_t *size_lim;
 
-	/**@brief Store constant values to set mem giver res_flag
+	/** Number of dimensions (if XDIM ndim = len(size_lim)-1) */
+	size_t ndim;
+
+	/**@brief Store constant values to set mem given res_flag
 	 * - For @ref RPN_IF_RES_CONST const_val points on a single value
 	 * - For @ref RPN_IF_RES_CONST_RGBA const_val points on 4 values
 	 * - Else const_val is set to NULL
@@ -94,72 +103,109 @@ struct rpn_if_default_data_s
  *
  * @param pos_flag Binary OR combination of RPN_IF_POSITION_*
  *     (@ref ifs_if_default_posflag )
- * @param pos_flag Binary OR combination of RPN_IF_RES_*
- *     (@ref ifs_if_default_posflag )
+ * @param res_flag Binary OR combination of RPN_IF_RES_*
+ *     (@ref ifs_if_default_resflag )
  * @param lim Depends on pos_flag parameter (
  *	see @ref rpn_if_default_data_s::size_lim )
- * @param val Depends on res_flag parameter (
+ * @param res_const Depends on res_flag parameter (
  *	see @ref rpn_if_default_data_s::const_val )
+ * @param rpn_stack_sz The size of the stack for expressions
  * @returns A new @ref rpn_if_param_t or NULL on error
- * @todo Implementation/testing
+ * @ingroup ifs_if_default
  */
 rpn_if_param_t* rpn_if_default_params(short pos_flag, short res_flag,
-	const size_t *lim, const rpn_value_t *val, unsigned char rpn_stack_sz);
+	const size_t *lim, const rpn_value_t *res_const,
+	unsigned char rpn_stack_sz);
 
-/**@brief Default argf function ( see @ref rpn_if_param_s.arg_f ) */
-int rpn_if_argf_default(rpn_if_t *rif, size_t pos, rpn_value_t *args);
-/**@brief Default result function ( see @ref rpn_if_param_s.res_f ) */
-int rpn_if_resf_default(rpn_if_t *rif, size_t *pos, rpn_value_t *data);
+/** Fetch size limit and const values array sizes given flag values
+ *  @param pos_flag (@ref ifs_if_default_posflag)
+ *  @param res_flag (@ref ifs_if_default_resflag)
+ *  @param sizes size limit array size and constant values array size
+ *  @return 0 or -1 if a flag is not valid
+ *  @warning returns 1 for size limit when XDIM position, but actually the
+ *  limit is given by the 1st number in the limit (example : [2,640,480],
+ *  [3,16,640,480], ...)
+ *  @todo replace short by int for sizes
+ */
+int rpn_if_sizes_from_flag(short pos_flag, short res_flag, short sizes[2]);
 
-/**@brief Set the first expression argument from position
+/**@brief Default argf function ( see @ref rpn_if_param_s.getarg_f )
+ *
+ * This function handle internal rif modification to set next arguments.
+ * It will call specialized function depending on
+ * @ref rpn_if_default_data_s.pos_flag and @ref rpn_if_default_data_s.res_flag
+ * @note The position is the first set of arguments and can be use to
+ * look for the other one
+ *
+ * @param rif Pointer on expressions
+ * @param pos The position
+ * @return 0 or -1 on error
+ */
+int rpn_if_getarg_default(rpn_if_t *rif, size_t pos);
+
+/**@brief Default result function ( see @ref rpn_if_param_s.setres_f )
+ *
+ * This function will store the result at given position and 
+ * It will call specialized function depending on
+ * @ref rpn_if_default_data_s.pos_flag and @ref rpn_if_default_data_s.res_flag
+ * @param rif Pointer on expressions
+ * @param pos Will be set to the resulting position
+ * @return 0 or -1 on error
+ */
+int rpn_if_setres_default(rpn_if_t *rif, size_t *pos);
+
+/**@brief Set the first expression argument from linear position
  * @param rif Expressions
  * @param pos Memory map offset
  * @param args pointer on expressions arguments
- * @note Other arguments are set using the generic @ref rpn_if_argf_default
+ * @return 0 or -1 on error
+ * @note Other arguments are set using the generic @ref rpn_if_getarg_default
  * function
  */
 int rpn_if_argf_linear(rpn_if_t *rif, size_t pos, rpn_value_t *args);
 /**@brief Transform 1st expression result to position 
  * @param rif Expressions
  * @param pos Pointer on resulting position in memory map
- * @param data Pointer on resulting data
- * @note Data from position fecth is done by generic @ref rpn_if_resf_default
+ * @return 0 or -1 on error
+ * @note Data from position fecth is done by generic @ref rpn_if_setres_default
  * function
  */
-int rpn_if_resf_linear(rpn_if_t *rif, size_t *pos, rpn_value_t *data);
+int rpn_if_resf_linear(rpn_if_t *rif, size_t *pos);
 
 /**@brief Set the 1st & 2nd argument from position
  * @param rif Expressions
  * @param pos Memory map offset
  * @param args pointer on expression arguments
- * @note Other arguments are set using the generic @ref rpn_if_argf_default
+ * @return 0 or -1 on error
+ * @note Other arguments are set using the generic @ref rpn_if_getarg_default
  * function
  */
 int rpn_if_argf_xy(rpn_if_t *rif, size_t pos, rpn_value_t *args);
 /**@brief Transform 1st and 2nd result into a memory map's offset
  * @param rif Expressions
  * @param pos Memory map offset pointer
- * @param data Pointer on resulting data
- * @note Data from position fetch is done by generic @ref rpn_if_resf_default
+ * @return 0 or -1 on error
+ * @note Data from position fetch is done by generic @ref rpn_if_setres_default
  * function
  */
-int rpn_if_resf_xy(rpn_if_t *rif, size_t *pos, rpn_value_t *data);
+int rpn_if_resf_xy(rpn_if_t *rif, size_t *pos);
 
 /**@brief Set X first arguments from position
  * @param rif Expressions
  * @param pos Memory map offset
  * @param args Pointer on expression arguments
- * @note Other arguments are set using the generic @ref rpn_if_argf_default
+ * @return 0 or -1 on error
+ * @note Other arguments are set using the generic @ref rpn_if_getarg_default
  * function
  */
 int rpn_if_argf_xdim(rpn_if_t *rif, size_t pos, rpn_value_t *args);
 /**@brief Transform X arguments into a memory map's offset
  * @param rif Expressions
  * @param pos Memory map offset pointer
- * @param data Pointer on resulting data
- * @note Data from position fetch is done by generic @ref rpn_if_resf_default
+ * @return 0 or -1 on error
+ * @note Data from position fetch is done by generic @ref rpn_if_setres_default
  * function
  */
-int rpn_if_resf_xdim(rpn_if_t *rif, size_t *pos, rpn_value_t *data);
+int rpn_if_resf_xdim(rpn_if_t *rif, size_t *pos);
 
 #endif

rpn_ifs.c

@@ -7,6 +7,7 @@ rpn_ifs_t* rpn_ifs_new(rpn_if_param_t *params, rpn_value_t *memmap)
 
 	if(!(res = malloc(sizeof(rpn_ifs_t))))
 	{
+		err = errno;
 		goto error;
 	}
 	bzero(res, sizeof(rpn_ifs_t));
@@ -25,6 +26,7 @@ rpn_ifs_t* rpn_ifs_new(rpn_if_param_t *params, rpn_value_t *memmap)
 			-1, 0);
 		if(res->mem == (void*)-1)
 		{
+			err = errno;
 			goto mmap_err;
 		}
 	}
@@ -32,10 +34,8 @@ rpn_ifs_t* rpn_ifs_new(rpn_if_param_t *params, rpn_value_t *memmap)
 	return res;
 
 	mmap_err:
-		err = errno;
 		free(res);
 	error:
-		err = errno;
 		errno = err;
 		return NULL;
 }
@@ -60,6 +60,72 @@ void rpn_ifs_free(rpn_ifs_t *rifs)
 	}
 }
 
+size_t rpn_ifs_set_if_count(rpn_ifs_t *rifs, size_t count, unsigned int *weights)
+{
+	errno = 0;
+
+	const size_t old_sz = rifs->if_sz;
+
+	// free old functions if old_sz > count
+	if(old_sz)
+	{
+		for(size_t i=old_sz-1; i>=count; i--)
+		{
+			rpn_if_free(rifs->rpn_if[i]);
+		}
+	}
+
+	void *tmp;
+
+	if(!(tmp = realloc(rifs->rpn_if, sizeof(rpn_if_t*)*count)))
+	{
+		return 0;
+	}
+	rifs->rpn_if = tmp;
+
+	if(!(tmp = realloc(rifs->weight, sizeof(unsigned int) * count)))
+	{
+		return 0;
+	}
+	rifs->weight = tmp;
+
+	rifs->flat_sz = rifs->params.rpn_sz * count;
+	if(!(tmp = realloc(rifs->flat_rpn, sizeof(rpn_expr_t*) * rifs->flat_sz)))
+	{
+		rifs->flat_sz = rifs->params.rpn_sz * old_sz;
+		return 0;
+	}
+	rifs->flat_rpn = tmp;
+	rifs->if_sz = count;
+
+
+	// init new functions if old_sz < count
+	for(size_t i=old_sz; i<count; i++)
+	{
+		rifs->rpn_if[i] = rpn_if_new(&(rifs->params), rifs->mem, NULL);
+		if(!rifs->rpn_if[i])
+		{
+			return 0;
+		}
+		for(size_t j=0; j<rifs->params.rpn_sz; j++)
+		{
+			const size_t flat_idx = (i*rifs->params.rpn_sz) + j;
+			rifs->flat_rpn[flat_idx] = &(rifs->rpn_if[i]->rpn[j]);
+		}
+	}
+
+
+
+	// set all weights
+	memcpy(rifs->weight, weights, sizeof(unsigned int) * count);
+
+	if(rpn_ifs_weight_update(rifs) < 0)
+	{
+		return 0;
+	}
+	return rifs->if_sz;
+}
+
 size_t rpn_ifs_add_if(rpn_ifs_t *rifs, unsigned int weight)
 {
 	size_t res, i, first_flat;
@@ -81,7 +147,7 @@ size_t rpn_ifs_add_if(rpn_ifs_t *rifs, unsigned int weight)
 
 	rifs->weight[rifs->if_sz] = weight;
 	//WRONG expr ARGUMENT !!!
-	rifs->rpn_if[rifs->if_sz] = rpn_if_new(&(rifs->params), rifs->mem);
+	rifs->rpn_if[rifs->if_sz] = rpn_if_new(&(rifs->params), rifs->mem, NULL);
 	if(!rifs->rpn_if[rifs->if_sz])
 	{
 		return 0;
@@ -101,7 +167,7 @@ size_t rpn_ifs_add_if(rpn_ifs_t *rifs, unsigned int weight)
 	rifs->if_sz++;
 	if(rpn_ifs_weight_update(rifs) < 0)
 	{
-		rpn_ifs_del_if(rifs, res); // don't attempt to ceck for errors..
+		rpn_ifs_del_if(rifs, res); // don't attempt to check for errors..
 		return 0;
 	}
 	return res;
@@ -132,8 +198,34 @@ int rpn_ifs_del_if(rpn_ifs_t *rifs, size_t if_idx)
 	return 0;
 }
 
-int rpn_ifs_run(rpn_ifs_t *rifs, size_t n)
+int rpn_ifs_run(rpn_ifs_t *rifs, size_t n, unsigned char *rnd)
 {
+	unsigned char *_rnd;
+	if(!rnd)
+	{
+		if(!(_rnd = malloc(sizeof(unsigned char)*n)))
+		{
+			return -1;
+		}
+		if(getrandom(_rnd, n, GRND_NONBLOCK) < 0)
+		{
+			int err = errno;
+			free(_rnd);
+			errno = err;
+			return -1;
+		}
+	}
+	else
+	{
+		_rnd = rnd;
+	}
+
+	for(size_t i=0; i<n; i++)
+	{
+		rpn_if_t *cur_if = rifs->if_proba[_rnd[i]];
+		assert(cur_if != NULL);
+		rifs->pos = rpn_if_step(cur_if, rifs->pos);
+	}
 	return 0;
 }
 
@@ -144,7 +236,7 @@ int rpn_ifs_weight_update(rpn_ifs_t *rifs)
 	rpn_if_t **proba;
 
 	proba = rifs->if_proba;
-	bzero(rifs->if_proba, sizeof(rpn_if_t*)*255);
+	bzero(rifs->if_proba, sizeof(*rifs->if_proba));
 
 	weight_sum = 0;
 
@@ -155,14 +247,22 @@ int rpn_ifs_weight_update(rpn_ifs_t *rifs)
 	j=0;
 	for(i=0; i < rifs->if_sz; i++)
 	{
-		max = rifs->weight[i] * 255 / weight_sum;
-		while(max)
+		max = rifs->weight[i] * 256 / weight_sum;
+		while(max && j < 256)
 		{
-			*proba = rifs->rpn_if[i];
+			proba[j] = rifs->rpn_if[i];
 			max--;
 			j++;
 		}
 	}
+	for(j=j; j<256; j++)
+	{
+		// dirty & quick fill with last value
+		/* @todo enhance the random choice resolution to match
+		 * more closely asked weights (uint16 vs actualt uint8 random
+		 * choice ? */
+		rifs->if_proba[j] = rifs->if_proba[j-1];
+	}
 	return 0;
 }
 
@@ -171,8 +271,10 @@ rpn_expr_t **rpn_ifs_flat_rpn(rpn_ifs_t *rifs)
 	return NULL;
 }
 
-int rpn_ifs_step(rpn_ifs_t *rifs)
+int rpn_ifs_step(rpn_ifs_t *rifs, unsigned char rnd)
 {
-	return 1;
+	rpn_if_t *cur_if = rifs->if_proba[rnd];
+	rifs->pos = rpn_if_step(cur_if, rifs->pos);
+	return 0;
 }
 

rpn_ifs.h

@@ -21,11 +21,17 @@
 
 #include "config.h"
 
+#include <assert.h>
 #include <string.h>
 
 #include "rpn_jit.h"
 #include "rpn_if.h"
 
+/**@file rpn_ifs.h
+ * @brief IFS header
+ * @ingroup ifs
+ */
+
 /**@defgroup ifs Iterated function system
  * @brief IFS implementation with RPN expressions
  *
@@ -64,9 +70,9 @@ struct rpn_ifs_s
 
 	/**@brief Stores the original chance of choosing corresponding IF */
 	unsigned int *weight;
-	/** @brief Stores an array of 255 pointers on IF allowing fast
-	 * random choice. Last ptr can be NULL*/
-	rpn_if_t *if_proba[255];
+	/** @brief Stores an array of 256 pointers on IF allowing fast
+	 * random choice.*/
+	rpn_if_t *if_proba[256];
 
 	/**@brief Stores the RPN expressions pointer of the IF contained in
 	 * the system */
@@ -90,6 +96,15 @@ rpn_ifs_t* rpn_ifs_new(rpn_if_param_t *params, rpn_value_t *memmap);
  */
 void rpn_ifs_free(rpn_ifs_t *rifs);
 
+/**@brief Set the number and weights of functions in the system
+ * @note Do not re-init existing function
+ * @param rifs The iterated function system
+ * @param count The number of functions in the system
+ * @param weights The function's weight
+ * @return The number of function in the system
+ */
+size_t rpn_ifs_set_if_count(rpn_ifs_t *rifs, size_t count, unsigned int *weights);
+
 /**@brief Add a new iterated function to the system
  * @param rifs The iterated function system
  * @param weight The new expression weight
@@ -111,9 +126,10 @@ int rpn_ifs_del_if(rpn_ifs_t *rifs, size_t if_idx);
  * @note Make n random choices and call corresponding IF
  * @param rifs The iterated function system
  * @param n consecutive IFS calls
- * @return 1 if error else 0
+ * @param seed prepopulated array of random bytes (if NULL one is generated)
+ * @return -1 if error else 0
  */
-int rpn_ifs_run(rpn_ifs_t *rifs, size_t n);
+int rpn_ifs_run(rpn_ifs_t *rifs, size_t n, unsigned char *rnd);
 
 /**@brief Updates the @ref rpn_ifs_s.if_proba array using
  * @ref rpn_ifs_s.if_proba values
@@ -130,9 +146,11 @@ int rpn_ifs_weight_update(rpn_ifs_t *rifs);
 rpn_expr_t **rpn_ifs_flat_rpn(rpn_ifs_t *rifs);
 
 /**@brief Randomly choose an IF and make a step updating ifs current posisition
+ * @param rifs The IF system
+ * @param seed A random byte
  * @return -1 on error else 0
  */
-int rpn_ifs_step(rpn_ifs_t *rifs);
+int rpn_ifs_step(rpn_ifs_t *rifs, unsigned char rnd);
 
 #endif
 

rpn_jit.c

@@ -68,7 +68,13 @@ int rpn_expr_reinit(rpn_expr_t* expr)
 		return -1;
 	}
 #endif
-	bzero(expr->code_map, expr->code_map_sz);
+	if(_rpn_expr_reset_map(expr)  < 0)
+	{
+		snprintf(expr->err_reason, 128, 
+			"Unable to re-init code map : %s", strerror(errno));
+		expr->state = RPN_ERROR;
+		return -1;
+	}
 	bzero(expr->stack, sizeof(unsigned long) * expr->stack_sz);
 	if(_rpn_expr_init_map(expr) < 0)
 	{
@@ -81,6 +87,37 @@ int rpn_expr_reinit(rpn_expr_t* expr)
 	return 0;
 }
 
+
+int rpn_expr_recompile(rpn_expr_t *expr, const char *code)
+{
+	if(rpn_expr_reinit(expr) < 0)
+	{
+		return -1;
+	}
+	return rpn_expr_compile(expr, code);
+}
+
+int rpn_expr_tokens_updated(rpn_expr_t* expr)
+{
+	if(rpn_expr_reinit(expr) < 0)
+	{
+		return -1;
+	}
+	if(_rpn_expr_compile_tokens(expr) < 0)
+	{
+		expr->state = RPN_ERROR;
+		return -1;
+	}
+	if(expr->expr)
+	{
+		free(expr->expr);
+	}
+	expr->expr = rpn_tokenized_expr(&expr->toks, 0);
+	expr->state = RPN_READY;
+	return 0;
+}
+
+
 int rpn_expr_compile(rpn_expr_t *expr, const char *code)
 {
 #ifdef DEBUG
@@ -102,7 +139,7 @@ int rpn_expr_compile(rpn_expr_t *expr, const char *code)
 	return _rpn_expr_compile_expr(expr);
 }
 
-int rpn_expr_untokenize(rpn_expr_t *expr, rpn_tokenized_t *tokens, char long_op)
+int rpn_expr_untokenize(rpn_expr_t *expr, const rpn_tokenized_t *tokens, char long_op)
 {
 	int err;
 	size_t i;
@@ -139,7 +176,7 @@ int rpn_expr_untokenize(rpn_expr_t *expr, rpn_tokenized_t *tokens, char long_op)
 		if(_rpn_expr_token_copy(expr, &(tokens->tokens[i])) < 0)
 		{
 			err = errno;
-			if(errno == EINVAL)
+			if(errno == EUCLEAN)
 			{
 				dprintf(2,
 "Fatal error, unknown token type : %d.\nMemory corruption ?\n",
@@ -173,6 +210,8 @@ int rpn_expr_untokenize(rpn_expr_t *expr, rpn_tokenized_t *tokens, char long_op)
 
 char* rpn_random(size_t op_sz, size_t args_count)
 {
+	const int BUFF_ALLOC = 4096;
+
 	double step;
 	size_t i, buff_sz, offset, rnd;
 	char *buff, *cur;
@@ -195,9 +234,9 @@ char* rpn_random(size_t op_sz, size_t args_count)
 
 	for(i=0; i<op_sz; i++)
 	{
-		if(buff_sz - offset < 21)
+		if(buff_sz - offset < BUFF_ALLOC / 4)
 		{
-			buff_sz += 40;
+			buff_sz += BUFF_ALLOC;
 			cur = realloc(buff, sizeof(char) * buff_sz);
 			if(!cur)
 			{
@@ -252,10 +291,167 @@ char* rpn_random(size_t op_sz, size_t args_count)
 			offset += nchr;
 		}
 	}
-	buff[offset] = '\0';
+	if(offset)
+	{
+		buff[offset-1] = '\0';
+	}
 	return buff;
 }
 
+size_t rpn_expr_serialize(rpn_expr_t* expr, void *buf, size_t buf_sz)
+{
+	const size_t total_sz = sizeof(rpn_expr_serial_t) + \
+			 (sizeof(rpn_token_t)*expr->toks.tokens_sz) + \
+			 (sizeof(rpn_value_t)*expr->stack_sz);
+
+	if(total_sz < buf_sz || !buf)
+	{
+		return total_sz;
+	}
+
+	bzero(buf, total_sz);
+
+	rpn_expr_serial_t *ser = buf;
+	rpn_token_t *tokens = (void*)buf + sizeof(*ser);
+	rpn_value_t *stack = (void*)(tokens + expr->toks.tokens_sz);
+
+	ser->token_sz = expr->toks.tokens_sz;
+	ser->stack_sz = expr->stack_sz;
+	ser->argc = expr->toks.argc;
+	ser->state = expr->state;
+	memcpy(ser->err_reason, expr->err_reason, sizeof(ser->err_reason));
+	memcpy(stack, expr->stack, expr->stack_sz);
+
+	for(size_t i=0; i < ser->token_sz; i++)
+	{
+		tokens[i].type = expr->toks.tokens[i].type;
+		switch(expr->toks.tokens[i].type)
+		{
+			case RPN_arg:
+				tokens[i].arg_n = expr->toks.tokens[i].arg_n;
+				break;
+			case RPN_val:
+				tokens[i].value = expr->toks.tokens[i].value;
+				break;
+			case RPN_op:
+				tokens[i].op_n = expr->toks.tokens[i].op_n;
+				break;
+			default:
+				// SHOULD NEVER APPEND !
+				errno = EINVAL;
+				goto err;
+		}
+	}
+
+	return total_sz;
+
+err:
+	return 0;
+}
+
+
+int rpn_expr_deserialize(rpn_expr_t* expr, const void *buf, size_t buf_sz)
+{
+	int err = EINVAL;
+	const rpn_expr_serial_t *ser = buf;
+
+	if(!expr)
+	{
+		errno = EINVAL;
+		return -1;
+	}
+
+	if(buf_sz < sizeof(rpn_expr_serial_t))
+	{
+		snprintf(expr->err_reason, 128,
+				"Given buffer is to small (%ld bytes) to deserialize", buf_sz);
+		err = EINVAL;
+		goto err;
+	}
+
+	const size_t total_sz = sizeof(rpn_expr_serial_t) + \
+				(sizeof(rpn_token_t)*ser->token_sz) + \
+				(sizeof(rpn_value_t)*ser->stack_sz);
+	if(buf_sz < total_sz)
+	{
+		snprintf(expr->err_reason, 128,
+				"Expected %ld bytes but %ld given",
+				total_sz, buf_sz);
+		err = EINVAL;
+		goto err;
+	}
+
+	rpn_token_t *tokens = (void*)buf + sizeof(*ser);
+	rpn_value_t *stack = (void*)(tokens + ser->token_sz);
+
+	if(rpn_expr_init(expr, ser->stack_sz, ser->argc) < 0)
+	{
+		err = EINVAL;
+		goto err;
+	}
+
+	expr->state = ser->state;
+	memcpy(expr->err_reason, ser->err_reason, 128);
+	memcpy(expr->stack, stack, sizeof(rpn_value_t)*ser->stack_sz);
+
+	expr->args_count = expr->toks.argc = ser->argc;
+
+	rpn_tokenized_t toks;
+	toks.argc = ser->argc;
+	toks.tokens_sz = ser->token_sz;
+	toks.tokens = malloc(sizeof(rpn_token_t)*ser->token_sz);
+	if(!toks.tokens)
+	{
+		err = errno;
+		snprintf(expr->err_reason, 128,
+				"Unable to allocate tokens : %s",
+				strerror(errno));
+		goto err_expr;
+	}
+	
+
+	for(size_t i=0; i < ser->token_sz; i++)
+	{
+		toks.tokens[i].type = tokens[i].type;
+		switch(tokens[i].type)
+		{
+			case RPN_val:
+				toks.tokens[i].value = tokens[i].value;
+				break;
+			case RPN_arg:
+				toks.tokens[i].arg_n = tokens[i].arg_n;
+				break;
+			case RPN_op:
+				toks.tokens[i].op_n = tokens[i].op_n;
+				toks.tokens[i].op = &(rpn_ops[tokens[i].op_n]);
+				break;
+			default:
+				snprintf(expr->err_reason, 128,
+						"Invalid token type encountered %d", tokens[i].type);
+				err = EINVAL;
+				goto err_expr;
+		}
+	}
+
+	if(rpn_expr_untokenize(expr, &toks, 0) < 0)
+	{
+		err = EINVAL;
+		goto err_expr;
+	}
+
+	expr->toks = toks;
+
+	return 0;
+
+err_expr:
+	rpn_expr_close(expr);
+err:
+	expr->state = RPN_ERROR;
+	errno = err;
+	return -1;
+}
+
+
 int _rpn_expr_compile_expr(rpn_expr_t* expr)
 {
 	rpn_tokenizer_t tokenizer;
@@ -280,7 +476,7 @@ int _rpn_expr_compile_expr(rpn_expr_t* expr)
 	{
 		if(_rpn_expr_token_copy(expr, token) < 0)
 		{
-			if(errno == EINVAL)
+			if(errno == EUCLEAN)
 			{
 				dprintf(2,
 "Fatal error, unknown token type : %d chr %ld.\nMemory corruption ?\n",
@@ -326,7 +522,7 @@ int _rpn_expr_compile_tokens(rpn_expr_t* expr)
 		token = &(expr->toks.tokens[i]);
 		if(_rpn_expr_token_copy(expr, token) < 0)
 		{
-			if(errno == EINVAL)
+			if(errno == EUCLEAN)
 			{
 				dprintf(2,
 "Fatal error, unknown token type : %d\nMemory corruption ?\n",
@@ -358,7 +554,7 @@ int _rpn_expr_compile_tokens(rpn_expr_t* expr)
 unsigned long rpn_expr_eval(rpn_expr_t *expr, unsigned long *args)
 {
 	rpn_run_f expr_run;
-	unsigned long int res;
+	rpn_value_t res;
 	if(expr->state == RPN_ERROR)
 	{
 		return 0;
@@ -407,6 +603,11 @@ int _rpn_expr_token_copy(rpn_expr_t *expr, rpn_token_t *token)
 			value = NULL;
 			break;
 		case RPN_arg:
+			if(expr->args_count <= token->arg_n)
+			{
+				errno = EINVAL;
+				return -1;
+			}
 			local_op.fun = &rpn_arg;
 			local_op.fun_sz = &(CODE_SZ(rpn_arg));
 			value = &(token->arg_n);
@@ -417,7 +618,7 @@ int _rpn_expr_token_copy(rpn_expr_t *expr, rpn_token_t *token)
 			value = &(token->value);
 			break;
 		default:
-			errno = EINVAL;
+			errno = EUCLEAN;
 			return -1;
 	}
 	if(_rpn_code_part_cpy(expr, local_op.fun, *(local_op.fun_sz),
@@ -500,3 +701,18 @@ int _rpn_expr_end_map(rpn_expr_t *expr)
 	return 0;
 }
 
+int _rpn_expr_reset_map(rpn_expr_t *expr)
+{
+	if(!expr->code_map_sz)
+	{
+		return _rpn_expr_init_map(expr);
+	}
+	if(mprotect(expr->code_map, expr->code_map_sz,
+			PROT_READ | PROT_WRITE))
+	{
+		return -1;
+	}
+	bzero(expr->code_map, expr->code_map_sz);
+	return 0;
+}
+

rpn_jit.h

@@ -100,6 +100,9 @@ typedef unsigned long (*rpn_run_f)(unsigned long, unsigned long*, void*);
  * @ingroup rpn */
 typedef struct rpn_expr_s rpn_expr_t;
 
+/** @brief Serialized expression form */
+typedef struct rpn_expr_serial_s rpn_expr_serial_t;
+
 /**@brief Stores RPN expression informations 
  * @ingroup rpn
  */
@@ -134,6 +137,18 @@ struct rpn_expr_s
 	char err_reason[128];
 };
 
+/**@brief Serialized form of an RPN expression
+ * @ingroup rpn
+ */
+struct rpn_expr_serial_s
+{
+	size_t token_sz;
+	unsigned char stack_sz;
+	size_t argc;
+	short state;
+	char err_reason[128];
+};
+
 /**@brief Initialize a new @ref rpn_expr_s
  * @param expr Pointer on the expression
  * @param stack_sz Expression stack size
@@ -148,10 +163,24 @@ int rpn_expr_init(rpn_expr_t* expr, const unsigned char stack_sz,
 
 /**@brief Reinit an existing @ref rpn_expr_s avoiding mmap and malloc
  * for executable memory map and for stack
- * @param expr
+ * @param expr Pointer on the expression
+ * @return 0 or -1 on error
  */
 int rpn_expr_reinit(rpn_expr_t* expr);
 
+/**@brief Recompile an existing, allready initialized, expression
+ * @param expr The expression
+ * @param code The code to compile
+ * @return 0 if no error else -1
+ */
+int rpn_expr_recompile(rpn_expr_t *expr, const char *code);
+
+/**@brief Takes into account modifications in token representation
+ * @param expr The expression  with updated tokens
+ * @return 0 if no error else -1
+ */
+int rpn_expr_tokens_updated(rpn_expr_t* expr);
+
 /**@brief Starts a new initialized expression from an expression string
  * @param expr Pointer on an intialized expression ( see @ref rpn_expr_init )
  * @param code '\0' terminated string representing the RPN expr
@@ -166,14 +195,15 @@ int rpn_expr_compile(rpn_expr_t *expr, const char *code);
  * @param tokens Tokenized form
  * @param long_op If true long operation are used, else 1 chr op are used
  * @return 0 if no error else -1 and @ref rpn_expr_s::err_reason is set
- * @note The @ref rpn_tokenized_s::tokens attribute is "captured" by the
+ *
+ * @note The @ref rpn_tokenized_s.tokens attribute is "captured" by the
  * @ref rpn_expr_s structure and will be deallocated when @ref rpn_expr_close
  * is called. It is not encouraged to keep a reference on this attribute after
- * @ref rpn_expr_start_untokenize call (but pointed @ref rpn_tokenized_t 
+ * @ref rpn_expr_untokenize call (but pointed @ref rpn_tokenized_t 
  * can be reused)
  * @ingroup rpn
  */
-int rpn_expr_untokenize(rpn_expr_t *expr, rpn_tokenized_t *tokens, char long_op);
+int rpn_expr_untokenize(rpn_expr_t *expr, const rpn_tokenized_t *tokens, char long_op);
 
 /**@brief Generate a new random rpn_expression
  * @param op_sz Number of token in generated expression
@@ -190,7 +220,7 @@ char* rpn_random(size_t op_sz, size_t args_count);
  * @ingroup rpn_compile
  */
 int _rpn_expr_compile_expr(rpn_expr_t* expr);
-/**@brief Compile an new RPN expression from string expression
+/**@brief Compile an new RPN expression from tokens
  * @param expr Pointer on @ref rpn_expr_s
  * @return 0 if no error else -1 and set @ref rpn_expr_s err_reason
  * @ingroup rpn_compile
@@ -208,7 +238,6 @@ unsigned long rpn_expr_eval(rpn_expr_t *expr, unsigned long *args);
 
 /**@brief Free ressources handled by given @ref rpn_expr_s
  * @param expr Pointer on rpn_expr_t
- * @param expr Pointer on @ref rpn_expr_s
  * @ingroup rpn
  */
 void rpn_expr_close(rpn_expr_t* expr);
@@ -219,6 +248,11 @@ void rpn_expr_close(rpn_expr_t* expr);
  */
 void rpn_expr_reset_stack(rpn_expr_t *expr);
 
+/**@todo document*/
+size_t rpn_expr_serialize(rpn_expr_t* expr, void *buf, size_t buf_sz);
+/**@todo document*/
+int rpn_expr_deserialize(rpn_expr_t* expr, const void *buf, size_t buf_sz);
+
 /**@brief Copy precompiled code from @ref rpn_lib.h in expression code map
  * @param expr The expression being compiled
  * @param token Pointer on token informations
@@ -255,4 +289,10 @@ int _rpn_expr_init_map(rpn_expr_t* expr);
  */
 int _rpn_expr_end_map(rpn_expr_t *expr);
 
+/**@brief Reset the memory map, filling it with zeroes and reseting permissions
+ * @param expr Pointer on rpn_expr_t
+ * @return 0 if no error else -1
+ */
+int _rpn_expr_reset_map(rpn_expr_t *expr);
+
 #endif

rpn_lib.asm

@@ -38,14 +38,18 @@
 	global %1_sz
 %endmacro
 
+section .note.GNU-stack noalloc noexec nowrite progbits
+
 section .data
 
 rpn_exec:
-	; unsigned long int rpn_exec(unsigned long int stack_size, unsigned long args)
+	; unsigned long int rpn_exec(unsigned long int stack_size, unsigned long args, unsigned long int *stack)
 	; rdi -> stack size (in values (64 bits))
 	; rsi -> args pointers
-	
+	; rdx -> stack pointer or null
+
 	enter 32, 0
+	push rbx ; rbx MUST BE PRESERVED !!! (like r12 to r15 and rbp !)
 	
 	mov stack_size, rdi
 	mov args_ptr, rsi
@@ -100,6 +104,7 @@ rpn_exec_ret:
 	syscall
 	pop rax
 	.end:
+	pop rbx
 	leave
 	ret
 part_sz rpn_exec_ret

rpn_lib.h

@@ -50,7 +50,9 @@
 #define CODE_SZ(NAME) NAME ## _sz
 
 /**@brief macro to declare a code part and associated size */
-#define CODE_PART(NAME) const extern void* NAME; extern const unsigned long NAME ## _sz
+#define CODE_PART(NAME) const extern void* NAME; \
+/**@brief Cord part size */\
+extern const unsigned long NAME ## _sz
 
 /**@brief Define the type of value manipulated by RPN expressions
  *
@@ -58,6 +60,11 @@
  * @todo use it */
 typedef unsigned long int rpn_value_t;
 
+/**@brief Small alias for PyLong_FromUnsignedLong */
+#define PyLong_FromRpnValue_t PyLong_FromUnsignedLong
+/**@brief Small alias for PyLong_AsUnsignedLong */
+#define PyLong_AsRpnValue_t PyLong_AsUnsignedLong
+
 /**@brief Function heading code
  *
  * - stack frame creation

rpn_mutate.c

@@ -0,0 +1,390 @@
+#include "rpn_mutate.h"
+
+const rnd_t rnd_t_max = -1;
+
+const rpn_mutation_params_t rpn_default_mutation = {
+	.min_len = 3,
+	.w_add = 1.25,
+	.w_del = 1.0,
+	.w_mut = 2.0,
+	.w_mut_soft = 4.0,
+	.w_add_elt = {1,1,1},
+	.w_mut_elt={1,1,1},
+};
+
+#define to_weight(total, elt) ((rnd_t)((((long double)elt)*rnd_t_max)/total))
+int rpn_mutation_init_params(rpn_mutation_params_t *params)
+{
+	long double total;
+
+	if(params->w_add < 0) { goto err; }
+	if(params->w_del < 0) { goto err; }
+	if(params->w_mut < 0) { goto err; }
+	if(params->w_mut_soft < 0) { goto err; }
+
+	total = params->w_add + params->w_del + params->w_mut + params->w_mut_soft;
+	params->_weights[0] = to_weight(total, params->w_add);
+	params->_weights[1] = to_weight(total, params->w_del);
+	params->_weights[2] = to_weight(total, params->w_mut);
+	params->_weights[3] = to_weight(total, params->w_mut_soft);
+/*
+#ifdef DEBUG
+dprintf(2, "weights : %d %d %d %d\n",
+		params->_weights[0],
+		params->_weights[1],
+		params->_weights[2],
+		params->_weights[3]);
+#endif
+*/
+	for(uint8_t i=1; i<4; i++)
+	{
+		params->_weights[i] += params->_weights[i-1];
+	}
+/*
+#ifdef DEBUG
+dprintf(2, "summed weights : %d %d %d %d\n",
+		params->_weights[0],
+		params->_weights[1],
+		params->_weights[2],
+		params->_weights[3]);
+#endif
+*/
+
+	total = 0;
+	for(uint8_t i=0; i<3; i++)
+	{
+		if(params->w_add_elt[i] < 0) { goto err; }
+		total += params->w_add_elt[i];
+	}
+	for(uint8_t i=0; i<3; i++)
+	{
+		rnd_t w;
+		if(total > 0)
+		{
+			w = to_weight(total, params->w_add_elt[i]);
+		}
+		else
+		{
+			w = to_weight(3,1);
+		}
+		params->_weights[i+4] = w;
+		if(i>0)
+		{
+			params->_weights[i+4] += params->_weights[i+3];
+		}
+	}
+
+	total = 0;
+	for(uint8_t i=0; i<3; i++)
+	{
+		if(params->w_mut_elt[i] < 0) { goto err; }
+		total += params->w_mut_elt[i];
+	}
+	for(uint8_t i=0; i<3; i++)
+	{
+		rnd_t w;
+		if(total > 0)
+		{
+			w = to_weight(total, params->w_mut_elt[i]);
+		}
+		else
+		{
+			w = to_weight(3, 1);
+		}
+		params->_weights[i+7] = w;
+		if(i>0)
+		{
+			params->_weights[i+7] += params->_weights[i+6];
+		}
+	}
+
+	return 0;
+err:
+	errno = EINVAL;
+	return -1;
+}
+#undef to_weight
+
+int rpn_mutation(rpn_tokenized_t *toks, rpn_mutation_params_t *params)
+{
+	//_print_params(2, params);
+	assert(toks->argc < rnd_t_max);
+
+	if(toks->tokens_sz <= params->min_len)
+	{
+		return rpn_mutation_add(toks, params);
+	}
+	size_t choice;
+	if(_rpn_random_choice(4, params->_weights, &choice) < 0)
+	{
+		return -1;
+	}
+	switch(choice)
+	{
+		case 0:
+			return rpn_mutation_add(toks, params);
+		case 1:
+			return rpn_mutation_del(toks, params);
+		case 2:
+			return rpn_mutation_mut(toks, params);
+		case 3:
+			return rpn_mutation_mut_soft(toks, params);
+		default:
+			dprintf(2, "Random error that should never occurs");
+			return -1;
+	}
+	
+}
+int rpn_mutation_add(rpn_tokenized_t *toks, rpn_mutation_params_t *params)
+{
+	rpn_token_t *new_toks, new_token;
+	size_t position;
+
+	if(toks->tokens_sz == 0)
+	{
+		position = 0;
+	}
+	else if(rpn_rand_limit(toks->tokens_sz, &position) < 0)
+	{
+		return -1;
+	}
+	
+	if(rpn_random_token_type(&(new_token.type), &params->_weights[4]) < 0)
+	{
+		return -1;
+	}
+	if(rpn_mutation_random_token(&new_token, toks, params) < 0)
+	{
+		return -1;
+	}
+
+	toks->tokens_sz++;
+	new_toks = realloc(toks->tokens, sizeof(rpn_token_t) * toks->tokens_sz);
+	if(!new_toks)
+	{
+		toks->tokens_sz--;
+		return -1;
+	}
+	toks->tokens = new_toks;
+
+	if(position < toks->tokens_sz - 1)
+	{
+		memmove(&(toks->tokens[position+1]), &(toks->tokens[position]),
+				sizeof(rpn_token_t) * ((toks->tokens_sz-1) - position));
+	}
+	memcpy(&toks->tokens[position], &new_token, sizeof(rpn_token_t));
+
+	return 0;
+}
+int rpn_mutation_del(rpn_tokenized_t *toks, rpn_mutation_params_t *params)
+{
+	rpn_token_t *new_toks;
+	size_t position;
+
+	if(rpn_rand_limit(toks->tokens_sz-1, &position) < 0)
+	{
+		return -1;
+	}
+
+	if(position != toks->tokens_sz - 1)
+	{
+		memmove(&(toks->tokens[position]), &(toks->tokens[position+1]),
+				sizeof(rpn_token_t) * ((toks->tokens_sz - 1) - position));
+	}
+	toks->tokens_sz--;
+	new_toks = realloc(toks->tokens, sizeof(rpn_token_t) * toks->tokens_sz);
+	if(!new_toks)
+	{
+		toks->tokens_sz = 0;
+		toks->tokens = NULL;
+		return -1;
+	}
+	toks->tokens = new_toks;
+	return 0;
+}
+int rpn_mutation_mut(rpn_tokenized_t *toks, rpn_mutation_params_t *params)
+{
+	rpn_token_t *tok;
+	size_t position;
+
+	if(rpn_rand_limit(toks->tokens_sz-1, &position) < 0)
+	{
+		return -1;
+	}
+
+	tok = &toks->tokens[position];
+
+	if(rpn_random_token_type(&(tok->type), &(params->_weights[7])) < 0)
+	{
+		return -1;
+	}
+	if(rpn_mutation_random_token(tok, toks, params) < 0)
+	{
+		return -1;
+	}
+
+	return 0;
+}
+int rpn_mutation_mut_soft(rpn_tokenized_t *toks, rpn_mutation_params_t *params)
+{
+	rpn_token_t *tok;
+	size_t position;
+
+	if(rpn_rand_limit(toks->tokens_sz-1, &position) < 0)
+	{
+		return -1;
+	}
+
+	tok = &(toks->tokens[position]);
+
+	if(rpn_mutation_random_token(tok, toks, params) < 0)
+	{
+		return -1;
+	}
+
+	return 0;
+}
+
+
+int rpn_mutation_random_token(rpn_token_t *tok,
+		rpn_tokenized_t *toks, rpn_mutation_params_t *params)
+{
+	rnd_t rand;
+
+	if(rpn_getrandom(&rand) < 0)
+	{
+		return  -1;
+	}
+
+	unsigned char op_n;
+	unsigned char *val;
+	size_t left;
+	switch(tok->type)
+	{
+		case RPN_op:
+			op_n = rand / (rnd_t_max/RPN_OP_SZ);
+			op_n %= RPN_OP_SZ;
+			tok->op_n = op_n;
+			tok->op = &(rpn_ops[op_n]);
+			break;
+
+		case RPN_arg:
+			tok->arg_n = rand / (rnd_t_max/toks->argc);
+			tok->arg_n %= toks->argc;
+			break;
+		case RPN_val:
+			val =(unsigned char*)&(tok->value);
+			left = sizeof(tok->value);
+			do
+			{
+				ssize_t ret = getrandom(val, left, GRND_NONBLOCK);
+				if(ret == -1)
+				{
+					if(errno == EAGAIN || errno == EINTR)
+					{
+						continue;
+					}
+					return -1;
+				}
+				left -= ret;
+				val += ret;
+			}while(left);
+			break;
+		default:
+			dprintf(2, "Another random error that shouldn't occur\n");
+			return -1;
+	}
+	return 0;
+}
+
+
+int rpn_random_token_type(rpn_token_type_t *type, rnd_t *weights)
+{
+	const rpn_token_type_t types[3]  = { RPN_op, RPN_arg, RPN_val };
+	size_t choice;
+
+	if(_rpn_random_choice(3, weights, &choice) < 0)
+	{
+		return -1;
+	}
+	*type = types[choice];
+	return 0;
+}
+
+int rpn_getrandom(rnd_t *rand)
+{
+	rnd_t seed;
+	char *seed_ptr = (char*)&seed;
+	size_t buflen = sizeof(seed);
+
+	do
+	{
+		ssize_t ret = getrandom(seed_ptr, buflen, GRND_NONBLOCK);
+		if(ret == -1)
+		{
+			if(errno == EAGAIN || errno == EINTR)
+			{
+				continue;
+			}
+			return -1;
+		}
+		buflen -= ret;
+		seed_ptr += ret;
+	}while(buflen);
+	*rand = seed;
+	return 0;
+}
+
+int _rpn_random_choice(size_t sz, rnd_t *weights, size_t *res)
+{
+	rnd_t rand;
+	if(rpn_getrandom(&rand) < 0) { return -1; }
+	__rpn_random_choice(sz, weights, rand, res);
+	return 0;
+}
+
+void __rpn_random_choice(size_t sz, rnd_t *weights, rnd_t rand, size_t *res)
+{
+	for(*res=0; *res<sz; (*res)++)
+	{
+		if(weights[*res] >= rand)
+		{
+			return;
+		}
+	}
+	*res = sz-1;
+}
+
+int rpn_rand_limit(size_t max, size_t *res)
+{
+	unsigned long int step = rnd_t_max / max;
+	rnd_t rnd;
+
+	if(rpn_getrandom(&rnd) < 0)
+	{
+		return -1;
+	}
+	for(*res=0; *res<max; (*res)++)
+	{
+		if(*res * step >= rnd)
+		{
+			return 0;
+		}
+	}
+	//*res = max - 1;
+	return 0;
+}
+
+void _print_params(int fd, rpn_mutation_params_t *params)
+{
+	dprintf(fd, "Minlen = %ld w[add=%.2f del=%.2f mut=%.2f msf=%.2f]\n",
+			params->min_len, params->w_add, params->w_del,
+			params->w_mut, params->w_mut_soft);
+	dprintf(fd, "w_add_elt [op=%.2f const=%.2f var=%.2f]\n",
+			params->w_add_elt[0], params->w_add_elt[1],
+			params->w_add_elt[2]);
+	dprintf(fd, "w_mut_elt [op=%.2f const=%.2f var=%.2f]\n",
+			params->w_mut_elt[0], params->w_mut_elt[1],
+			params->w_mut_elt[2]);
+}
+

rpn_mutate.h

@@ -0,0 +1,158 @@
+/*
+ * Copyright (C) 2020,2023 Weber Yann
+ * 
+ * This file is part of pyrpn.
+ * 
+ * pyrpn is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * 
+ * pyrpn is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with pyrpn.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __rpn_mutate__h__
+#define __rpn_mutate__h__
+#include "config.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <stdint.h>
+#include <sys/random.h>
+
+#include "rpn_parse.h"
+
+/**@file rpn_mutate.h
+ * @brief Expression mutation headers
+ */
+
+/**@brief Mutation parameters */
+typedef struct rpn_mutation_params_s rpn_mutation_params_t;
+/**@brief Type of random values used by mutations */
+typedef uint16_t rnd_t;
+
+extern const rnd_t rnd_t_max;
+
+/** @brief Mutation parameters */
+struct rpn_mutation_params_s
+{
+	/**@brief Minimum expression length  */
+	size_t min_len;
+	/**@brief Weight of adding a token*/
+	float w_add;
+	/**@brief Weight of deleting a token*/
+	float w_del;
+	/**@brief Weight of mutating a token*/
+	float w_mut;
+	/**@brief Weight of mutating a token without type change*/
+	float w_mut_soft;
+	/**@brief Weight for each token types (op, const, var) when
+	 * adding a token*/
+	float w_add_elt[3];
+	/**@brief Weight for each token types (op, const, var) when
+	 * mutating a token*/
+	float w_mut_elt[3];
+
+	/**@brief For internal use, set by rpn_mutation_init_params
+	 *
+	 * weight reported by groups on rnd_t integers
+	 * - [0..3] -> weights mutation
+	 * - [4..6] -> w_add_elt
+	 * - [7..9] -> w_mut_elt
+	 *
+	 * @note Weights are stored a way allowing fast random choice.
+	 * They are kind of ascending threshold until @ref rnd_t max value.
+	 */
+	rnd_t  _weights[10];
+};
+
+/**@brief Initialize mutation parameters 
+ *
+ * pre-process integers threshold by group
+ * @param params
+ * @return 0 if no error else -1 and set ERRNO (EINVAL)
+ */
+int rpn_mutation_init_params(rpn_mutation_params_t *params);
+
+/**@brief Mutate a tokenized rpn expression
+ * @param toks The tokenized expression
+ * @param params The mutation parameters
+ * @return 0 or -1 on error */
+int rpn_mutation(rpn_tokenized_t *toks, rpn_mutation_params_t *params);
+
+/**@brief Mutate an expression by adding a token
+ * @param toks The tokenized expression
+ * @param params The mutation parameters
+ * @return 0 or -1 on error */
+int rpn_mutation_add(rpn_tokenized_t *toks, rpn_mutation_params_t *params);
+/**@brief Mutate an expression by removing a token
+ * @param toks The tokenized expression
+ * @param params The mutation parameters
+ * @return 0 or -1 on error */
+int rpn_mutation_del(rpn_tokenized_t *toks, rpn_mutation_params_t *params);
+/**@brief Mutate an expression by changing a token
+ * @param toks The tokenized expression
+ * @param params The mutation parameters
+ * @return 0 or -1 on error */
+int rpn_mutation_mut(rpn_tokenized_t *toks, rpn_mutation_params_t *params);
+/**@brief Mutate an expression by changing a token value (not type)
+ * @param toks The tokenized expression
+ * @param params The mutation parameters
+ * @return 0 or -1 on error */
+int rpn_mutation_mut_soft(rpn_tokenized_t *toks, rpn_mutation_params_t *params);
+
+/**@brief Change the "value" of a token randomly not it's type
+ * @param tok Point on the token to mutate
+ * @param toks The tokenized expression the token tok belongs to
+ * @param params The mutation parameters
+ * @return 0 or -1 on error */
+int rpn_mutation_random_token(rpn_token_t *tok,
+		rpn_tokenized_t *toks, rpn_mutation_params_t *params);
+
+
+/**@brief Choose a random token type
+ * @param type Point on result
+ * @param weights The weight of each types for the choice
+ * @note Weights comes from @ref rpn_mutation_params_s._weights and are
+ * processed in a form allowing a fast random choice.
+ * @return 0 or -1 on error
+ */
+int rpn_random_token_type(rpn_token_type_t *type, rnd_t *weights);
+
+/**@brief Generate a random value
+ * @param rand Point on result
+ * @return -1 on error else 0 */
+int rpn_getrandom(rnd_t *rand);
+
+/**@brief Generate a random number between 0 and max (max excluded)
+ * @param max Maximum value
+ * @param res Will be set to a value between [..max[
+ * @return -1 on error else 0 */
+int rpn_rand_limit(size_t max, size_t *res);
+
+/**@brief Given a size return an random element with regards to given weights
+ * @param sz The given size
+ * @param weights Weights coming from @ref rpn_mutation_params_s._weights
+ * @param res Points on result
+ * @return 0 or -1 on error */
+int _rpn_random_choice(size_t sz, rnd_t *weights, size_t *res);
+
+/**@brief Given a size and a (random) value, returns an element with regards
+ * to given weights 
+ * @param sz The given size
+ * @param weights Weights coming from @ref rpn_mutation_params_s._weights
+ * @param rand A (random) value
+ * @param res Points on result */
+void __rpn_random_choice(size_t sz, rnd_t *weights, rnd_t rand, size_t *res);
+
+/**@brief Debugging function that dump mutation params in a human readable format
+ * @param fd The file descriptor on wich we should print parameters
+ * @param params The mutation parameters to dump */
+void _print_params(int fd, rpn_mutation_params_t *params);
+
+#endif

rpn_mutation.c

@@ -1,58 +0,0 @@
-/*
- * Copyright (C) 2020 Weber Yann
- * 
- * This file is part of pyrpn.
- * 
- * pyrpn is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * any later version.
- * 
- * pyrpn is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with pyrpn.  If not, see <http://www.gnu.org/licenses/>.
- */
-#include "rpn_mutation.h"
-
-/**@file rpn_mutation.c
- * @todo continue implementation */
-
-const rpn_mutation_profile_t rpn_default_mutprof = {16,
-	{	RPN_del, RPN_del,
-		RPN_add, RPN_add,
-		RPN_chg, RPN_chg, RPN_chg, RPN_chg, RPN_chg, RPN_chg,
-		RPN_upd, RPN_upd, RPN_upd, RPN_upd, RPN_upd, RPN_upd}
-};
-
-rpn_expr_t* rpn_expr_mutation(rpn_expr_t *src, size_t mutations)
-{
-	return rpn_expr_mutation_p(src, mutations, &rpn_default_mutprof);
-}
-
-rpn_expr_t* rpn_expr_mutation_p(rpn_expr_t *src, size_t mutations,
-		const rpn_mutation_profile_t *prof)
-{
-	unsigned char op;
-	
-	prof = prof?prof:&rpn_default_mutprof;
-
-	op = prof->mods[(int)(drand48() / (1.0 / prof->mods_sz))];
-	switch(op)
-	{
-		case 0: // add a token
-			break;
-		case 1: // delete a token
-			break;
-		case 2: // update token type
-			break;
-		default: // update token, same type
-			break;
-	}
-	return NULL;
-}
-
-

rpn_mutation.h

@@ -1,83 +0,0 @@
-/*
- * Copyright (C) 2020 Weber Yann
- * 
- * This file is part of pyrpn.
- * 
- * pyrpn is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * any later version.
- * 
- * pyrpn is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with pyrpn.  If not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef __rpn_mutation__h__
-#define __rpn_mutation__h__
-
-#include <stddef.h>
-
-#include "rpn_jit.h"
-
-/**@defgroup mutation RPN expression mutation
- * @ingroup rpn
- */
-/**@file rpn_mutation.h
- * @brief Contains structures and function to mutate RPN epxressions
- */
-
-
-/**@brief Defines mutation actions types */
-enum rpn_mutation_op_e {
-	/**@brief Mutation action : delete a token */
-	RPN_del,
-	/**@brief Mutation action : add a token */
-	RPN_add,
-	/**@brief Mutation action : change a token */
-	RPN_chg,
-	/**@brief Mutation action : update a token (same type, different value) */
-	RPN_upd
-};
-
-/**@brief Shortcut for struct @ref rpn_mutation_profile_s */
-typedef struct rpn_mutation_profile_s rpn_mutation_profile_t;
-
-/**@brief Stores mutation informations
- * @ingroup mutation
- */
-struct rpn_mutation_profile_s
-{
-	/**@brief Size of @ref rpn_mutation_profile_s::mods attribute */
-	size_t mods_sz;
-
-	/**@brief Modification possibilities
-	 *
-	 * One value is picked up randomly from this list to determine
-	 * the type of mutation : addition, deletion, modification, value change
-	 */
-	unsigned char mods[];
-};
-
-/**@brief Default mutation profile */
-extern const rpn_mutation_profile_t rpn_default_mutprof;
-
-/**@brief Shortcut for @ref rpn_expr_mutation_p with a @ref rpn_default_mutprof
- * @ingroup mutation */
-rpn_expr_t* rpn_expr_mutation(rpn_expr_t *src, size_t mutations);
-
-/**@brief Generate a new expression by applying mutations to a source
- * expression
- * @param src Source expression
- * @param mutations number of mutations
- * @param prof Mutation profile
- * @return A new instance of rpn_expr_t ready to be evaluate
- * @ingroup mutation
- */
-rpn_expr_t* rpn_expr_mutation_p(rpn_expr_t *src, size_t mutations,
-	const rpn_mutation_profile_t *prof);
-
-#endif

rpn_parse.c

@@ -42,6 +42,7 @@ const rpn_op_t rpn_ops[] = {\
 	__op(rpn_pop_op, 'p', "pop"),\
 };
 #undef __op
+const size_t RPN_OP_SZ = (sizeof(rpn_ops) / sizeof(rpn_op_t));
 
 int rpn_tokenizer_start(rpn_tokenizer_t *tokenizer, rpn_tokenized_t *dst,
 	const char* expr, size_t argc)
@@ -213,7 +214,7 @@ void rpn_tokenizer_free(rpn_tokenizer_t *tokenizer)
 	}
 }
 
-char* rpn_tokenized_expr(rpn_tokenized_t *tokens, char long_op)
+char* rpn_tokenized_expr(const rpn_tokenized_t *tokens, char long_op)
 {
 	size_t expr_sz, i;
 	int err, written;
@@ -244,7 +245,7 @@ char* rpn_tokenized_expr(rpn_tokenized_t *tokens, char long_op)
 	for(i=0; i<tokens->tokens_sz; i++)
 	{
 		token = &(tokens->tokens[i]);
-		if(cur - expr >= expr_sz - ALLOC_CHUNK)
+		if((size_t)(cur - expr) >= (expr_sz - ALLOC_CHUNK))
 		{
 			expr_sz += 128;
 			tmp = realloc(expr, sizeof(char) * expr_sz);
@@ -303,7 +304,11 @@ char* rpn_tokenized_expr(rpn_tokenized_t *tokens, char long_op)
 		#endif
 		cur += written;
 	}
-
+	
+	if(cur > expr)
+	{
+		*(cur-1) = '\0';
+	}
 	return expr;
 
 	free_err:
@@ -323,6 +328,15 @@ const rpn_op_t* rpn_match_token(const char* token)
 	return &(rpn_ops[rep]);
 }
 
+const rpn_op_t* rpn_op_from_opcode(unsigned char opcode)
+{
+	if(opcode > RPN_OP_SZ)
+	{
+		return NULL;
+	}
+	return &(rpn_ops[opcode]);
+}
+
 int rpn_match_token_i(const char* token)
 {
 	unsigned char i;
@@ -386,6 +400,28 @@ int rpn_match_number(const char* token, unsigned long *result)
 	return 0;
 }
 
+int rpn_token_snprintf(rpn_token_t *tok, char *dst, size_t sz)
+{
+	switch(tok->type)
+	{
+		case RPN_op:
+			if(tok->op->chr)
+			{
+				return snprintf(dst, sz, "%c", tok->op->chr);
+			}
+			return snprintf(dst, sz, "%s", tok->op->str);
+		case RPN_val:
+			return snprintf(dst, sz, "0x%lX", tok->value);
+		case RPN_arg:
+			return snprintf(dst, sz, "A%lu", tok->arg_n);
+		default:
+			errno = EINVAL;
+			return -1;
+	}
+
+}
+
+
 size_t rpn_op_sz()
 {
 	return sizeof(rpn_ops)/sizeof(rpn_op_t);

rpn_parse.h

@@ -37,7 +37,7 @@
  * @brief Parsing an expression into a @ref rpn_tokenized_t
  * 
  * The tokenized form ( see @ref rpn_tokenized_t ) of an expression is usefull
- * for @ref mutation.
+ * for mutations (see @ref rpn_mutate.h ).
  *
  * The tokenizing process is done in a way allowing compilation process to
  * fetch tokens while parsing the expression (see @ref rpn_tok).
@@ -49,7 +49,7 @@
  */
 
 /**@brief Shortcut for loop on all operations list */
-#define foreach_rpn_ops(IDX) for(IDX=0; IDX<rpn_op_sz(); IDX++)
+#define foreach_rpn_ops(IDX) for(IDX=0; IDX<RPN_OP_SZ; IDX++)
 
 /**@brief Check if a tokenizer is in error state
  * @param tokenizer Pointer on a @ref rpn_tokenizer_s
@@ -92,7 +92,7 @@ enum rpn_token_type_e {
 	/**@brief The token is an argument */
 	RPN_arg,
 	/**@brief The token is a value */
-	RPN_val
+	RPN_val,
 };
 
 /**@brief Represent an expression token (value, argument or operation)
@@ -165,6 +165,10 @@ struct rpn_tokenizer_s
  * @ingroup rpn_tokenize */
 extern const rpn_op_t rpn_ops[];
 
+/**@brief The count of operand (the size of @ref rpn_ops array) */
+extern const size_t RPN_OP_SZ;
+
+
 /**@brief Initialize a tokenizer and a tokenized representation
  * @param tokenizer Pointer on a new tokenizer
  * @param dst Pointer on a tokenized struct to store generated tokens
@@ -205,13 +209,13 @@ void rpn_tokenizer_free(rpn_tokenizer_t *tokenizer);
  */
 int rpn_tokenize(const char *token, rpn_token_t *dst, char error[64]);
 
-/**@brief Represented a tokenized expression in a string
+/**@brief Represent a tokenized expression in a string
  * @param tokens Tokenized expression
  * @param long_op If true uses @ref rpn_op_s::str else @ref rpn_op_s::chr
  * @return A newly allocated char* that should be deallocated using free()
  * @ingroup rpn_tokenize
  */
-char* rpn_tokenized_expr(rpn_tokenized_t *tokens, char long_op);
+char* rpn_tokenized_expr(const rpn_tokenized_t *tokens, char long_op);
 
 /**@brief Returns NULL or a pointer on corresponding operation infos
  * @param token The token we want to match
@@ -219,6 +223,13 @@ char* rpn_tokenized_expr(rpn_tokenized_t *tokens, char long_op);
  * @ingroup rpn_parse
  */
 const rpn_op_t* rpn_match_token(const char* token);
+
+/**@brief Returns NULL or pointer on corresponding operation infos
+ * @param opcode (index in @ref rpn_ops )
+ * @return NULL or operation informations
+ */
+const rpn_op_t* rpn_op_from_opcode(unsigned char opcode);
+
 /**@brief Return -1 or an index corresponding to @ref rpn_ops
  * @param token The token we want to match
  * @return NULL or operation informations
@@ -235,10 +246,20 @@ int rpn_match_token_i(const char* token);
  */
 int rpn_match_number(const char* token, unsigned long *result);
 
+/**@brief Stores a token string representation in given buffer
+ * @param token The token to represent
+ * @param dst The destination buffer for the string
+ * @param sz The biffer size
+ * @return Same as snprintf (the number of chr stored or negative value on error
+ */
+int rpn_token_snprintf(rpn_token_t *token, char *dst, size_t sz);
+
 /**@brief Get operations list size
  * @return number of operations in @ref rpn_ops
  */
 size_t rpn_op_sz();
+/**@brief Macro version of @ref rpn_op_sz() */
+#define RPN_OPS_SZ (sizeof(rpn_ops)/sizeof(rpn_op_t))
 
 /**@page rpn_lang RPN expression syntax
  * @brief Howto write an expression
@@ -274,7 +295,7 @@ size_t rpn_op_sz();
  * Each valid operations are declared in @ref rpn_ops variable (see 
  * @ref rpn_parse.c for details).
  *
- * The @ref python_module expose a function pyrpn.get_ops() ( @see pyrpn_ops )
+ * The @ref pymod_pyrpn expose a function pyrpn.get_ops() ( @see pyrpn_ops )
  * returning a dict with long operations as key and short as value.
  * \subsubsection rpn_lan_op_internal Internal mechanism
  * Operations are done using a loopstack : operands are poped from stack, and

send_live.example

@@ -0,0 +1 @@
+while [ 1 ]; do for img in ../rpnifs2_*.png; do convert "$img" -alpha off  "$(basename $img)"; echo -n '.'; done; echo ""; rsync -c -v  ./rpnifs2_*.png root@http.zvirt:/home/www-data/www/rpnifs/live/; sleep 40; done

tests/Makefile

@@ -8,16 +8,16 @@ DEPS=$(wildcard ../*.o)
 all: checks
 
 checks: $(BINARIES) $(SOURCES)
-	for test_bin in $(BINARIES); do echo "Running $${test_bin}.c"; ./$$test_bin && echo "OK" || echo "fail"; done
+	for test_bin in $(BINARIES); do echo "Running $${test_bin}.c"; ./$$test_bin && echo "OK" || false; done;
 
 ../%.o:
-	make -C ..
+	$(MAKE) -C .. `basename "$@"`
 
-%.o: %.c
-	$(CC) -I.. $(CFLAGS) -c -o $@ $<
+%.o: %.c $(DEPS)
+	$(CC) -I .. $(CFLAGS) -c -o $@ $<
 
-test_%: test_%.o ../rpn_lib.o ../rpn_jit.o ../rpn_parse.o ../rpn_if.o ../rpn_if_default.o
-	$(CC) -I..  $(CFLAGS) -o $@ $^
+test_rpn: test_rpn.o ../rpn_lib.o ../rpn_jit_cov.o ../rpn_parse_cov.o ../rpn_if_cov.o ../rpn_if_default_cov.o
+	$(CC) -I..  $(CFLAGS) -o $@ $^ -lgcov
 	
 .PHONY: clean
 

tests/benchmark.py

@@ -1,10 +1,15 @@
 #!/usr/bin/python3
 
+import argparse
+import os
 import sys
 import random
 import time
 import warnings
 
+def usage():
+    print("Usage : %s [expr_count [max_iter [argc [sz]]]]" %sys.argv[0],
+          file=sys.stderr)
 
 try:
     import pyrpn
@@ -13,14 +18,24 @@ except (ImportError, NameError) as e:
           file=sys.stderr)
     raise e
 
-#expr_count = 0x200
-#max_iter = 0x3000
-#argc = 2
-#sz = 0x20
-expr_count = 5000
-max_iter = 0x5000
-argc = 2
-sz = 0x30
+allint = lambda val: int(val, 0)
+
+parser = argparse.ArgumentParser(description="Benchmark RPNExpr vs IFS")
+parser.add_argument("-c", "--expr-count", type=allint, default=0x200)
+parser.add_argument("-i", "--iterations", type=allint, default=0x5000)
+parser.add_argument("-s", "--expr-size", type=allint, default=0x50)
+parser.add_argument("-G", "--gnuplot-output", type=str, default=None)
+
+args = parser.parse_args()
+expr_count = args.expr_count
+max_iter = args.iterations
+sz = args.expr_size
+
+argc = 1
+
+gpout=None
+if args.gnuplot_output is not None:
+    gpout = open(args.gnuplot_output, "a")
 
 try:
     from tqdm import tqdm
@@ -33,14 +48,6 @@ except (ImportError, NameError) as e:
     tqr = range
     write=True
 
-if len(sys.argv) > 1:
-    expr_count = int(sys.argv[1], 0)
-if len(sys.argv) > 2:
-    max_iter = int(sys.argv[2], 0)
-if len(sys.argv) > 3:
-    sz = int(sys.argv[4], 0)
-if len(sys.argv) > 4:
-    argc = int(sys.argv[3], 0)
 
 tot = 0
 time_op = 0
@@ -48,20 +55,27 @@ start = time.time()
 IMAX = (1<<63)-1
 samples = 8
 
-print("Running %dK iter on %d expressions with %d op" % (max_iter//1000,
+print("========\nIF  :", end=" ")
+print("Running %dK iter on %d expressions with %d op and %d args" % (max_iter//1000,
                                                          expr_count,
-                                                         sz))
+                                                         sz, argc))
 
+res = [0 for _ in range(512)]
 rnd_samples = [[[random.randint(0, IMAX) for _ in range(argc)] for _ in range(max_iter)]
                for _ in range(samples)]
+
 for i in tqr(expr_count):
     rnd_expr = pyrpn.random_expr(argc, sz)
     all_start = time.time()
     expr = pyrpn.RPNExpr(rnd_expr, argc)
+    expr.eval(*[0 for _ in range(argc)])
     start_op = time.time()
     rnds = random.choice(rnd_samples)
     for rnd in rnds:
-        expr.eval(*rnd)
+        r = expr.eval(*rnd)
+        # emulate memory access for simple if to make comparison
+        # consistant with IFS
+        res[r%512] += 1
     time_op += time.time() - start_op
     tot += time.time() - all_start
     if write:
@@ -70,9 +84,59 @@ for i in tqr(expr_count):
 if write:
     sys.stderr.write("\n")
 total = time.time() - start
-print("Runned %dK iter on %d expressions in %.2fs" % (max_iter//1000,
-                                                     expr_count, total))
-msg = "%.1fK iter/s %.1fms per expr (%dK iter, %d op per expr)"
-msg %= (max_iter*expr_count/time_op/1000, tot/expr_count*1000,
+
+ops = max_iter*expr_count*sz/time_op/(1000*1000)
+msg = "%5.1fM op/s %6.1fK iter/s %.1fms per expr (%dK iter, %d op per expr)"
+msg %= (ops,
+        max_iter*expr_count/time_op/1000,
+        tot/expr_count*1000,
         max_iter//1000, sz)
 print(msg)
+print("++++++++")
+if_ops = ops
+
+
+if_sz = 5
+ifs_count = expr_count // if_sz
+ifs_sz = 3
+mem_sz = (512,512)
+expr_count = ifs_count * if_sz
+
+print("IFS :", end=" ")
+print("Running %dK iter on %d ifs (sz=%d with %d if choosen randomly and %d op) RGB of size %r" % \
+        (max_iter/1000, ifs_count , if_sz, ifs_sz, sz, mem_sz))
+
+ifs = pyrpn.RPNIFS(pyrpn.const.POS_XY, pyrpn.const.RESULT_RGB, mem_sz)
+runtime = 0
+all_start = time.time()
+for i in tqr(ifs_count):
+    weights = [random.randint(1,50) for _ in range(ifs_sz)]
+    ifs.weights(weights)
+    for i in range(ifs_sz):
+        for expr_id in 'RGBXY':
+            ifs[i][expr_id] = pyrpn.random_expr(if_sz, sz)
+
+    rnd = os.getrandom(max_iter)
+    start = time.time()
+    ifs.run(max_iter, rnd)
+    runtime += time.time() - start
+    if write:
+        sys.stderr.write(".")
+        sys.stderr.flush()
+if write:
+    sys.stderr.write("\n")
+
+total = time.time() - all_start
+ops = max_iter*ifs_count*if_sz*sz/(runtime*1000*1000)
+msg = "%5.1fM op/s %6.1fK expr_iter/s %.2fms per ifs"
+msg %= (ops,
+        max_iter*ifs_count*if_sz/runtime/1000,
+        total/ifs_count)
+ifs_ops = ops
+print(msg)
+print("========")
+
+if gpout:
+    line = "%d %d %f %f\n" % (max_iter, sz, if_ops, ifs_ops)
+    gpout.write(line)
+    gpout.close()

tests/pyrpn.so

@@ -1 +0,0 @@
-../pyrpn.so

tests/test_rpn.c

@@ -113,6 +113,18 @@ int test_add()
 			res);
 		return 2;
 	}
+	
+	// test token update
+	expr.toks.tokens[0].value = 15;
+	rpn_expr_tokens_updated(&expr);
+	res = rpn_expr_eval(&expr, NULL);
+	//printf("Result = %ld\n", res);
+	if(res != 41)
+	{
+		dprintf(2, "Error : expected 42 but %ld received\n",
+			res);
+		return 3;
+	}
 	rpn_expr_close(&expr);
 	return 0;
 }
@@ -243,15 +255,15 @@ int test_rpn_if_default()
 		return -1;
 	}
 	
-	rif = rpn_if_new(rif_param, NULL);
+	rif = rpn_if_new(rif_param, NULL, NULL);
 	if(!rif)
 	{
 		perror("rpn_if_new() failed");
 		return -1;
 	}
 
-	free(rif_param);
 	rpn_if_free(rif);
+	free(rif_param);
 
 	return 0;
 }
@@ -270,15 +282,15 @@ int test_rpn_if_default2()
 		return -1;
 	}
 	
-	rif = rpn_if_new(rif_param, NULL);
+	rif = rpn_if_new(rif_param, NULL, NULL);
 	if(!rif)
 	{
 		perror("rpn_if_new() failed");
 		return -1;
 	}
 
-	free(rif_param);
 	rpn_if_free(rif);
+	free(rif_param);
 
 	return 0;
 }

tests/tests_pyrpn.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python3
-# Copyright 2020 Weber Yann
+# Copyright 2020, 2023 Weber Yann
 #
-# This file is part of geneifs.
+# This file is part of rpnifs.
 #
 #        geneifs is free software: you can redistribute it and/or modify
 #        it under the terms of the GNU General Public License as published by
@@ -17,6 +17,7 @@
 #        along with geneifs.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import copy
 import sys
 import random
 import math
@@ -34,28 +35,6 @@ except (ImportError, NameError) as e:
           file=sys.stderr)
     raise e
 
-## TODO : move this function somewhere or implement it in C
-def decode_state(state):
-    """ Return a dict containing rpn state dump fields
-    """
-    res = dict()
-    res['real_sz'] = len(state)
-    res['total_sz'] = int.from_bytes(state[0:8], byteorder="little")
-    res['argc'] =  int.from_bytes(state[8:16], byteorder="little")
-    res['stack_sz_bytes'] = state[16:24]
-    res['stack_sz'] = state[16]
-    res['token_sz'] = int.from_bytes(state[24:32], byteorder="little", signed=True)
-    res['stack'] = [int.from_bytes(state[i:i+8], byteorder="little")
-                    for i in range(32, 32 + (8*res['stack_sz']),8)]
-    res['tokens_off'] = 32 + (8*res['stack_sz'])
-    res['tokens_real_sz'] = res['real_sz'] - res['tokens_off']
-    res['tokens'] = [state[i:i+24]
-                      for i in range(res['tokens_off'],
-                                     res['tokens_off'] + (24*res['token_sz']),
-                                     24)]
-    return res
-    pass
-
 
 class Test0RpnModule(unittest.TestCase):
 
@@ -65,10 +44,11 @@ class Test0RpnModule(unittest.TestCase):
 
     def test_init_badargs(self):
         """ RPNExpr instanciation with bad arguments """
-        badargs = [('', 2), (), ('ab+',), ('ab+',300), (42, 42), ('ab', '42')]
+        badargs = [(), ('ab+',), ('ab+',300), (42, 42), ('ab', '42')]
         for badarg in badargs:
-           with self.assertRaises((ValueError, TypeError)):
-            expr = pyrpn.RPNExpr(*badarg)
+           with self.subTest(badargs=badarg):
+             with self.assertRaises((ValueError, TypeError)):
+              expr = pyrpn.RPNExpr(*badarg)
 
     def test_init_loop(self):
         """ Testing pyrpn.RPNExpr multiple instanciation """
@@ -112,6 +92,15 @@ class Test0RpnModule(unittest.TestCase):
             self.assertIn(long, known_ops)
             self.assertEqual(short, known_ops[long])
 
+    def testing_rand_expr_len(self):
+        """ Testing if RPNExpr.random_expr() returns expected expression """
+        for _ in range(100):
+            elen = random.randint(1, 50)
+            rnd_expr = pyrpn.random_expr(20, elen)
+            spl = [s for s in rnd_expr.split(' ') if len(s.strip())]
+            with self.subTest(expected_tokens=elen, tokens_count=len(spl), expr=rnd_expr):
+                self.assertEqual(len(spl), elen)
+
     def test_rand_expr(self):
         """ Testing RPNExpr.random_expr() """
         result = {}
@@ -137,31 +126,11 @@ class Test0RpnModule(unittest.TestCase):
                         if tok not in counters:
                             counters[tok] = 0
                         counters[tok] += 1
-        all_ops = len(pyrpn.get_ops()) + 2
+        all_ops = len(pyrpn.get_ops()) + 1
         entropy = 1-sum([(n/all_count)**2
                           for _, n in counters.items()])
         self.assertGreater(entropy, 1-(1/all_ops), "Low entropy !")
-    
-    def test_pickle_state(self):
-        """ Testing pickling/unpickling """
-        e = pyrpn.RPNExpr('0x42 + A0', 2)
-        e2 = pickle.loads(pickle.dumps(e))
-        ds_e = decode_state(e.__getstate__())
-        ds_e2 = decode_state(e2.__getstate__())
-        self.assertEqual(e.__getstate__(), e2.__getstate__(),
-                         msg="EXPR: %r != %r (%r != %r)" % (e, e2, ds_e, ds_e2))
-        for i in range(100):
-            e = pyrpn.RPNExpr(pyrpn.random_expr(0), 2)
-
-            e2 = pickle.loads(pickle.dumps(e))
-
-            self.assertEqual(e.__getstate__(), e2.__getstate__(),
-                             msg="EXPR#%d : %r != %r" % (i,e, e2))
-
-            e3 = pickle.loads(pickle.dumps(e2))
-
-            self.assertEqual(e.__getstate__(), e3.__getstate__(), msg=e)
-            self.assertEqual(e2.__getstate__(), e3.__getstate__(), msg=e)
+ 
 
 if __name__ == '__main__':
     unittest.main()

tests/tests_rpn_compile.py

@@ -34,6 +34,8 @@ except (ImportError, NameError) as e:
           file=sys.stderr)
     raise e
 
+from utils import Progress, VERBOSE
+
 class TestRpnCompile(unittest.TestCase):
 
     def test_basic(self):
@@ -55,11 +57,9 @@ class TestRpnCompile(unittest.TestCase):
 
     def test_long_code(self):
         """ Compile longs expressions (from 256 to 65536 op )"""
-        for i in range(0x100,0x10000,0x500):
+        for i in Progress(range(0x100,0x10000,0x500)):
             with self.subTest('Testing expression with %X ops' % i):
                 for argc in range(1,32, 8):
-                    sys.stderr.write('.')
-                    sys.stderr.flush()
                     args = [random.randint(0,IMAX) for _ in range(argc)]
                     expr = pyrpn.RPNExpr(pyrpn.random_expr(argc, i), argc)
                     del(expr)
@@ -67,18 +67,21 @@ class TestRpnCompile(unittest.TestCase):
     def test_very_long(self):
         """ Compile 3 very long expression (5242K op) """
         argc = 4
-        codelen = 0x500000
+        codelen = 0x50000
         import time
-        for i in range(3):
+        for i in Progress(3):
             args = [random.randint(0,IMAX) for _ in range(argc)]
-            sys.stderr.write('Generate')
-            sys.stderr.flush()
+            if VERBOSE:
+                sys.stderr.write('Generate')
+                sys.stderr.flush()
             expr_str = pyrpn.random_expr(argc, codelen)
-            sys.stderr.write('d Compile')
-            sys.stderr.flush()
+            if VERBOSE:
+                sys.stderr.write('d Compile')
+                sys.stderr.flush()
             expr = pyrpn.RPNExpr(expr_str, argc)
-            sys.stderr.write('d ')
-            sys.stderr.flush()
+            if VERBOSE:
+                sys.stderr.write('d ')
+                sys.stderr.flush()
             del(expr)
 
     def test_pickling(self):  
@@ -89,8 +92,10 @@ class TestRpnCompile(unittest.TestCase):
             pik = pickle.dumps(expr)
             new_expr = pickle.loads(pik)
             self.assertEqual(str(expr), str(new_expr))
+            pik2 = pickle.dumps(new_expr)
             args = [random.randint(0,IMAX) for _ in range(argc)]
             self.assertEqual(expr.eval(*args), new_expr.eval(*args))
+            self.assertEqual(pik, pik2)
 
 
 if __name__ == '__main__':

tests/tests_rpn_copy.py

@@ -0,0 +1,110 @@
+#!/usr/bin/python3
+# Copyright 2023 Weber Yann
+#
+# This file is part of rpnifs.
+#
+#        geneifs is free software: you can redistribute it and/or modify
+#        it under the terms of the GNU General Public License as published by
+#        the Free Software Foundation, either version 3 of the License, or
+#        (at your option) any later version.
+#
+#        geneifs is distributed in the hope that it will be useful,
+#        but WITHOUT ANY WARRANTY; without even the implied warranty of
+#        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#        GNU General Public License for more details.
+#
+#        You should have received a copy of the GNU General Public License
+#        along with geneifs.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import copy
+import pickle
+import sys
+
+import unittest
+
+
+try:
+    import pyrpn
+except (ImportError, NameError) as e:
+    print("Error importing pyrpn. Try to run make.",
+          file=sys.stderr)
+    raise e
+
+
+## TODO : move this function somewhere or implement it in C
+def decode_state(state):
+    """ Return a dict containing rpn state dump fields
+    """
+    res = dict()
+    res['real_sz'] = len(state)
+    res['total_sz'] = int.from_bytes(state[0:8], byteorder="little")
+    res['argc'] =  int.from_bytes(state[8:16], byteorder="little")
+    res['stack_sz_bytes'] = state[16:24]
+    res['stack_sz'] = state[16]
+    res['token_sz'] = int.from_bytes(state[24:32], byteorder="little", signed=True)
+    res['stack'] = [int.from_bytes(state[i:i+8], byteorder="little")
+                    for i in range(32, 32 + (8*res['stack_sz']),8)]
+    res['tokens_off'] = 32 + (8*res['stack_sz'])
+    res['tokens_real_sz'] = res['real_sz'] - res['tokens_off']
+    res['tokens'] = [state[i:i+24]
+                      for i in range(res['tokens_off'],
+                                     res['tokens_off'] + (24*res['token_sz']),
+                                     24)]
+    return res
+
+
+
+class TestRpnExprState(unittest.TestCase):
+    """ Testing RPNExpr "state" related methods (__copy__, __get/setstate__)
+    """
+
+    def test_pickle_state(self):
+        """ Testing pickling/unpickling """
+        e = pyrpn.RPNExpr('0x42 + A0', 2)
+        e2 = pickle.loads(pickle.dumps(e))
+        ds_e = decode_state(e.__getstate__())
+        ds_e2 = decode_state(e2.__getstate__())
+        self.assertEqual(e.__getstate__(), e2.__getstate__(),
+                         msg="EXPR: %r != %r (%r != %r)" % (e, e2, ds_e, ds_e2))
+        for i in range(100):
+            e = pyrpn.RPNExpr(pyrpn.random_expr(0), 2)
+            e.mutate(n_mutations=15);
+
+            e2 = pickle.loads(pickle.dumps(e))
+
+            self.assertEqual(e.__getstate__(), e2.__getstate__(),
+                             msg="EXPR#%d : %r != %r" % (i,e, e2))
+
+            e3 = pickle.loads(pickle.dumps(e2))
+
+            self.assertEqual(e.__getstate__(), e3.__getstate__(), msg=e)
+            self.assertEqual(e2.__getstate__(), e3.__getstate__(), msg=e)
+
+
+
+    def test_copy(self):
+        """ Basic copy test based on pickling """
+        e = pyrpn.RPNExpr('0x42 + A0', 2)
+        e2 = copy.copy(e)
+        ds_e = decode_state(e.__getstate__())
+        ds_e2 = decode_state(e2.__getstate__())
+        self.assertEqual(e.__getstate__(), e2.__getstate__(),
+                         msg="EXPR: %r != %r (%r != %r)" % (e, e2, ds_e, ds_e2))
+        for i in range(100):
+            e = pyrpn.RPNExpr(pyrpn.random_expr(0), 2)
+
+            e2 = copy.copy(e)
+
+            self.assertEqual(e.__getstate__(), e2.__getstate__(),
+                             msg="EXPR#%d : %r != %r" % (i,e, e2))
+
+            e3 = pickle.loads(pickle.dumps(e2))
+
+            self.assertEqual(e.__getstate__(), e3.__getstate__(), msg=e)
+            self.assertEqual(e2.__getstate__(), e3.__getstate__(), msg=e)
+
+
+if __name__ == '__main__':
+    unittest.main()
+

tests/tests_rpn_eval.py

@@ -34,6 +34,8 @@ except (ImportError, NameError) as e:
           file=sys.stderr)
     raise e
 
+from utils import Progress
+
 class TestRpnEval(unittest.TestCase):
 
     def test_arithm(self):
@@ -50,10 +52,8 @@ class TestRpnEval(unittest.TestCase):
             #('l', '<<'),
             #('r', '>>')
         ]
-        for rpn_op, pyop in ops:
+        for rpn_op, pyop in Progress(ops):
             with self.subTest('Testing op %s (%s)' % (rpn_op, pyop)):
-                sys.stderr.write('.')
-                sys.stderr.flush()
                 for i in range(0x1000):
                     op1, op2 = random.randint(0,IMAX), random.randint(1,IMAX)
                     pyexpr = '%d %s %d' % (op1, pyop, op2)
@@ -117,7 +117,7 @@ class TestRpnEval(unittest.TestCase):
         self.assertEqual(res, 0)
 
 
-    def test_airthm_extended(self):
+    def test_arithm_extended(self):
         """ Extended arithmetic tests """
         exprs = (
             ('A0 A1 +', '{0} + {1}', 2),

tests/tests_rpn_ifs.py

@@ -0,0 +1,290 @@
+#!/usr/bin/python3
+# copyright 2023 weber yann
+#
+# this file is part of rpnifs.
+#
+#        geneifs is free software: you can redistribute it and/or modify
+#        it under the terms of the gnu general public license as published by
+#        the free software foundation, either version 3 of the license, or
+#        (at your option) any later version.
+#
+#        geneifs is distributed in the hope that it will be useful,
+#        but without any warranty; without even the implied warranty of
+#        merchantability or fitness for a particular purpose.  see the
+#        gnu general public license for more details.
+#
+#        you should have received a copy of the gnu general public license
+#        along with geneifs.  if not, see <http://www.gnu.org/licenses/>.
+#
+
+import copy
+import mmap
+import pickle
+import random
+import sys
+
+import unittest
+
+try:
+    import numpy as np
+except (ImportError, NameError) as e:
+    np = None
+    warnings.warn("Numpy not found, some tests skipped", warnings.ImportWarning)
+
+def skipIfNoNumpy():
+    if np is not None:
+        return lambda func: func
+    return unittest.skip("Numpy not found")
+
+try:
+    import pyrpn
+except (ImportError, NameError) as e:
+    print("error importing pyrpn. try to run make.",
+          file=sys.stderr)
+    raise e
+
+from utils import *
+
+class TestRPNIFS(unittest.TestCase):
+    """ Testing RPNIFS features """
+
+    def test_init(self):
+        ifs = pyrpn.RPNIFS(pyrpn.const.POS_XY,
+                                pyrpn.const.RESULT_COUNT,
+                                (640,480))
+        self.assertEqual(len(ifs), 0)
+
+    def test_weights_set(self):
+        ifs = pyrpn.RPNIFS(pyrpn.const.POS_XY,
+                                pyrpn.const.RESULT_COUNT,
+                                (640,480))
+        self.assertEqual(len(ifs), 0)
+        w = ifs.weights([1,2,4])
+        self.assertEqual(len(ifs), 3)
+        self.assertEqual(w, (1,2,4))
+
+        w = ifs.weights()
+        self.assertEqual(w, (1,2,4))
+
+        for ifs_len in [random.randint(1,10) for _ in range(10)]:
+            for _ in range(10):
+                w = [random.randint(1,1000) for _ in range(ifs_len)]
+                ifs.weights(w)
+                self.assertEqual(len(ifs), ifs_len)
+                nw = ifs.weights()
+                self.assertEqual(tuple(w), nw)
+
+    def test_position(self):
+        ifs = pyrpn.RPNIFS(pyrpn.const.POS_XY,
+                                pyrpn.const.RESULT_COUNT,
+                                (640,480))
+        self.assertEqual(len(ifs), 0)
+        self.assertEqual(ifs.position(), 0)
+        self.assertEqual(ifs.position(10), 10)
+        self.assertEqual(ifs.position(), 10)
+        for i in range(640*480):
+            ifs.position(i)
+            self.assertEqual(ifs.position(), i)
+
+
+    def test_run(self):
+        """ Testing ifs run """
+        ifs = pyrpn.RPNIFS(pyrpn.const.POS_XY,
+                           pyrpn.const.RESULT_COUNT,
+                           (256,256))
+        ifs.weights([1])
+        ifs[0]['X'] = 'A0 13 +'
+        ifs[0]['Y'] = 'A1 2 +'
+        xpos = lambda steps: (steps*13)%256
+        ypos = lambda steps: (steps*2)%256
+        ifs.run(1)
+        self.assertEqual(ifs.position(), ifs.to_pos(xpos(1),ypos(1)))
+        steps = 1
+        for _ in range(128):
+            rnd = random.randint(1,128)
+            steps += rnd
+            ifs.run(rnd)
+            self.assertEqual(ifs.position(),
+                    ifs.to_pos(xpos(steps), ypos(steps)))
+
+
+    def test_run_rnd(self):
+        """ Testing ifs run with random if choice, same weights """
+        sz = [10000,10000]
+        ifs = pyrpn.RPNIFS(pyrpn.const.POS_XY,
+                           pyrpn.const.RESULT_COUNT,
+                           sz)
+        ifs.weights([1,1])
+        ifs[0]['X'] = 'A0 1 +'
+        ifs[0]['Y'] = 'A1 1 -'
+        ifs[1]['X'] = 'A0 1 -'
+        ifs[1]['Y'] = 'A0 1 +'
+        for _ in Progress(256):
+            ifs.position(ifs.to_pos(*[s//2 for s in sz]))
+            
+            steps = 0
+            steps_per_loop = 0x1000
+            for _ in range(4):
+                steps += steps_per_loop
+                ifs.run(steps_per_loop)
+                coord = ifs.from_pos(ifs.position())
+                
+                distance = [min(c, sz[i]-c) for i, c in enumerate(coord)]
+                dist = sum(distance)/2
+                percent = steps / (dist * 100)
+                self.assertLess(percent, 1)
+
+    @skipIfNoNumpy()
+    def test_buffer_protocol(self):
+        """ Testing buffer protocol on ifs's mmaps """
+        sz = [512,512]
+        ifs = pyrpn.RPNIFS(pyrpn.const.POS_XY,
+                           pyrpn.const.RESULT_COUNT,
+                           sz)
+        ifs.weights([1,2])
+        for rif in ifs:
+            for expr in rif:
+                expr.mutate(n_mutations=20)
+
+        buf = np.frombuffer(ifs, dtype=np.uint64)
+        buf2 = np.frombuffer(ifs[0], dtype=np.uint64)
+
+        for _ in range(512):
+            pos = random.randint(0, len(buf))
+            buf[pos] = random.randint(0, 0xFFFFFFFF)
+            pos = random.randint(0, len(buf))
+            buf2[pos] = random.randint(0, 0xFFFFFFFF)
+
+        self.assertTrue((buf == buf2).all())
+
+class TestRPNIFSCoordinates(unittest.TestCase):
+    """ Testing methods for coordinates <-> position convertions """
+
+    def test_position_linear(self):
+        """ Testing linear coordinate convertion methods """
+        rif = pyrpn.RPNIFS(pyrpn.const.POS_LINEAR,
+                    pyrpn.const.RESULT_CONST,
+                    (256,), (42,))
+
+        for pos in range(256):
+            with self.subTest(rif=rif, position=pos, expt=(pos,)):
+                res = rif.from_pos(pos)
+                self.assertEqual(res, (pos,))
+            with self.subTest(rif=rif, expt=pos, coord=(pos,)):
+                res = rif.to_pos(pos)
+                self.assertEqual(res, pos)
+
+    def test_position_xy(self):
+        """ Testing XY coordinate convertion methods """
+        for _ in Progress(5):
+            sz = (random.randint(32,128), random.randint(32,128))
+            rif = pyrpn.RPNIFS(pyrpn.const.POS_XY,
+                    pyrpn.const.RESULT_CONST,
+                    sz, (42,))
+            for pos in range(sz[0]*sz[1]):
+                coord = rif.from_pos(pos)
+                expt = (pos % sz[0], pos // sz[0])
+                with self.subTest(sz=sz, pos=pos, expt=expt, result=coord):
+                    self.assertEqual(expt, coord)
+                result = rif.to_pos(*coord)
+                with self.subTest(sz=sz, coord=coord, expt=pos, result=result):
+                    self.assertEqual(result, pos)
+
+    def test_position_xy_neg(self):
+        """ Testing XY coordinates with negative values """
+        sz = (100,200)
+        rif = pyrpn.RPNIFS(pyrpn.const.POS_XY,
+                pyrpn.const.RESULT_CONST,
+                sz, (42,))
+        
+        for _ in range(1000):
+            ny = random.randint(-0x1000000, sz[1])
+            nx = random.randint(0,sz[0]-1)
+            pos = rif.to_pos(nx, ny)
+            coord = rif.from_pos(pos)
+            self.assertEqual(coord, (nx, (ny%(1<<64))%sz[1]))
+
+    def test_position_xy_random(self):
+        """ Testing XY coordinate convertion methods (random samples)"""
+        for _ in Progress(256):
+            sz = (random.randint(32,4096), random.randint(32,4096))
+            rif = pyrpn.RPNIFS(pyrpn.const.POS_XY,
+                    pyrpn.const.RESULT_CONST,
+                    sz, (42,))
+            for _ in range(500):
+                pos = random.randint(0, (sz[0]*sz[1])-1)
+                coord = rif.from_pos(pos)
+                expt = (pos % sz[0], pos // sz[0])
+                with self.subTest(sz=sz, pos=pos, expt=expt, result=coord):
+                    self.assertEqual(expt, coord)
+                result = rif.to_pos(*coord)
+                with self.subTest(sz=sz, coord=coord, expt=pos, result=result):
+                    self.assertEqual(result, pos)
+
+    def test_position_xdim(self):
+        """ Testing XDIM coordinate convertion methods """
+        ndim = 5
+        resol = [8 for _ in range(ndim)]
+        sz = [ndim]+resol
+        linear_size = 1
+        for e in resol:
+            linear_size *= e
+        rif = pyrpn.RPNIFS(pyrpn.const.POS_XDIM,
+                pyrpn.const.RESULT_CONST,
+                sz, (42,))
+        for pos in range(linear_size):
+            coord = rif.from_pos(pos)
+            expt = []
+            cur = pos
+            for dim in resol:
+                elt = cur % dim
+                cur //= dim
+                expt.append(elt)
+            expt = tuple(expt)
+            with self.subTest(sz=sz, pos=pos, expt=expt, result=coord):
+                self.assertEqual(expt, coord)
+            result = rif.to_pos(*coord)
+            with self.subTest(sz=sz, coord=coord, expt=pos, result=result):
+                self.assertEqual(result, pos)
+
+    def test_position_xdim_neg(self):
+        """ Testing XDIM negative position convertion """
+        # TODO test negativ for other coordinate systems
+        ndim = 4
+        szlim = (ndim, 10,20,30,40)
+        rif = pyrpn.RPNIFS(pyrpn.const.POS_XDIM,
+                pyrpn.const.RESULT_CONST,
+                szlim, (42,))
+
+        pos = rif.to_pos(0,0,-5,0)
+        self.assertEqual(rif.from_pos(pos), (0,0,(-5%(1<<64))%30, 0))
+
+    def test_position_xdim(self):
+        """ Testing XDIM coordinate convertion methods (random sampling)"""
+        for _ in Progress(16):
+            ndim = random.randint(3,8)
+            resol = [random.randint(2, 16) for _ in range(ndim)]
+            sz = [ndim]+resol
+            linear_size = 1
+            for e in resol:
+                linear_size *= e
+            rif = pyrpn.RPNIFS(pyrpn.const.POS_XDIM,
+                    pyrpn.const.RESULT_CONST,
+                    sz, (42,))
+            for _ in range(2000):
+                pos = random.randint(0, linear_size-1)
+                coord = rif.from_pos(pos)
+                expt = []
+                cur = pos
+                for dim in resol:
+                    elt = cur % dim
+                    cur //= dim
+                    expt.append(elt)
+                expt = tuple(expt)
+                with self.subTest(sz=sz, pos=pos, expt=expt, result=coord):
+                    self.assertEqual(expt, coord)
+                result = rif.to_pos(*coord)
+                with self.subTest(sz=sz, coord=coord, expt=pos, result=result):
+                    self.assertEqual(result, pos)
+
+

tests/tests_rpn_mutate.py

@@ -0,0 +1,100 @@
+#!/usr/bin/python3
+# Copyright 2020, 2023 Weber Yann
+#
+# This file is part of rpnifs.
+#
+#        geneifs is free software: you can redistribute it and/or modify
+#        it under the terms of the GNU General Public License as published by
+#        the Free Software Foundation, either version 3 of the License, or
+#        (at your option) any later version.
+#
+#        geneifs is distributed in the hope that it will be useful,
+#        but WITHOUT ANY WARRANTY; without even the implied warranty of
+#        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#        GNU General Public License for more details.
+#
+#        You should have received a copy of the GNU General Public License
+#        along with geneifs.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import random
+import unittest
+
+
+try:
+    from tqdm import tqdm
+except (ImportError, NameError) as e:
+    tqdm = lambda i, *args, **kwargs:iter(i)
+
+try:
+    import pyrpn
+except (ImportError, NameError) as e:
+    print("Error importing pyrpn. Try to run make.",
+          file=sys.stderr)
+    raise e
+
+## TODO : test w_add_elt and w_mut_elt
+
+class TestRpnExprMutation(unittest.TestCase):
+
+    def test_base_mutation(self):
+        """ Testing mutation without parameters """
+        expr = pyrpn.RPNExpr("", 3)
+        expr.mutate()
+        expr.mutate(n_mutations=10)
+
+    def test_mutation_params_len(self):
+        """ Testing some value for parameters checking resulting expr len """
+
+        tests = (
+            ([1,1,0,0,0,(0,0,0),(0,0,0)], [1,0,100]),
+            ([1,2,1,0,0,(0,0,0),(0,0,0)], [1/3]),
+            ([1,2,0,1,0,(0,0,0),(0,0,0)], [2/3]),
+            ([1,3,1,2,0,(0,0,0),(0,0,0)], [2/6]),
+        )
+
+        for p_args, f_args in tests:
+            params = pyrpn.RPNMutationParamsTuple(p_args)
+            with self.subTest(params=params):
+                self._generic_mutation_params_lentest(params, *f_args)
+
+    def test_random_int_mutation_params(self):
+        """ Testing int random values for parameters checking resulting expr len """
+        for _ in range(50):
+            m_params = [random.randint(1,20) for _ in range(4)]
+            expt_len = (m_params[0]-m_params[1])/sum(m_params)
+            expt_len = 0 if expt_len < 0 else expt_len
+            params = [1] + m_params + [(0,0,0),(0,0,0)]
+            params = pyrpn.RPNMutationParamsTuple(params)
+            with self.subTest(params=params):
+                self._generic_mutation_params_lentest(params, expt_len)
+
+    def test_random_float_mutation_params(self):
+        """ Testing float random values for parameters checking resulting expr len """
+        for _ in range(50):
+            m_params = [random.randint(0,20)/20 for _ in range(4)]
+            expt_len = (m_params[0]-m_params[1])/sum(m_params)
+            expt_len = 0 if expt_len < 0 else expt_len
+            params = [1] + m_params + [(0,0,0),(0,0,0)]
+            params = pyrpn.RPNMutationParamsTuple(params)
+            with self.subTest(params=params):
+                self._generic_mutation_params_lentest(params, expt_len)
+
+    def _generic_mutation_params_lentest(self, params, expt_len, percent_error=.05,n_mut=5000):
+
+        expt_len = int(expt_len*n_mut)
+
+        expr = pyrpn.RPNExpr("", 3)
+        expr.mutate(n_mut, params)
+        msg = "Expected a len of %d ( [%d..%d] with %.1f%% error) but got %d"
+        self.assertTrue(abs(len(expr) - expt_len) <= n_mut*percent_error,
+                msg=msg % (expt_len,
+                           expt_len + n_mut*percent_error,
+                           expt_len - n_mut*percent_error,
+                           percent_error*100,
+                           len(expr)))
+
+
+if __name__ == '__main__':
+    unittest.main()
+

tests/tests_rpn_sequence.py

@@ -0,0 +1,121 @@
+#!/usr/bin/python3
+# copyright 2023 weber yann
+#
+# this file is part of rpnifs.
+#
+#        geneifs is free software: you can redistribute it and/or modify
+#        it under the terms of the gnu general public license as published by
+#        the free software foundation, either version 3 of the license, or
+#        (at your option) any later version.
+#
+#        geneifs is distributed in the hope that it will be useful,
+#        but without any warranty; without even the implied warranty of
+#        merchantability or fitness for a particular purpose.  see the
+#        gnu general public license for more details.
+#
+#        you should have received a copy of the gnu general public license
+#        along with geneifs.  if not, see <http://www.gnu.org/licenses/>.
+#
+
+import copy
+import pickle
+import random
+import sys
+
+import unittest
+
+try:
+    import pyrpn
+except (importerror, nameerror) as e:
+    print("error importing pyrpn. try to run make.",
+          file=sys.stderr)
+    raise e
+
+
+class TestRpnExprCopy(unittest.TestCase):
+    """ Testing RPNExpr sequence method """
+
+
+    def test_len(self):
+        """ Testing __len__ on some expressions """
+
+        tests = (
+            (['0x42 + A0', 2], 3),
+            (['A0', 10], 1),
+            (['', 1], 0),
+        )
+
+        for args, elen in tests:
+            e = pyrpn.RPNExpr(*args)
+            with self.subTest(expr=e, expected_len=elen):
+                self.assertEqual(len(e), elen)
+
+
+    def test_rand_len(self):
+        """ Testing __len__ with random expressions """
+        for _ in range(100):
+            elen = random.randint(1,50)
+            rexpr = pyrpn.random_expr(30,elen)
+            expr = pyrpn.RPNExpr(rexpr, 30)
+            with self.subTest(expr=expr, expected_len=elen):
+                self.assertEqual(len(expr), elen)
+
+    def test_getter(self):
+        """ Test __getitem__ """
+        tests = ((['0x42', '+', 'A0'], 1),
+                 (['A0'], 1))
+        for test in tests:
+            elts, nargs = test
+            expr_str = ' '.join(elts)
+            expr = pyrpn.RPNExpr(expr_str, nargs)
+            with self.subTest(expr=expr):
+                for i in range(len(expr)):
+                    self.assertEqual(expr[i],
+                            pyrpn.tokens.Token.from_str(elts[i]))
+
+    def test_setter(self):
+        """ Testing __setitem__ """
+        expr = pyrpn.RPNExpr('A0 A1 +', 2)
+        for _ in range(1024):
+            a0, a1 = (random.randint(0, 0x10000) for _ in range(2))
+            res = expr.eval(a0, a1)
+            self.assertEqual(res, a0+a1)
+
+        expr[2] = pyrpn.tokens.Operand('*')
+        self.assertEqual(str(expr), 'A0 A1 *')
+        for _ in range(1024):
+            a0, a1 = (random.randint(0, 0x1000) for _ in range(2))
+            res = expr.eval(a0, a1)
+            self.assertEqual(res, a0*a1)
+
+    def test_setter_append(self):
+        """ Testing __setitem__ to append new token """
+        expr = pyrpn.RPNExpr('A0 A1', 2)
+        expr[2] = pyrpn.tokens.Operand('+')
+        for _ in range(1024):
+            a0, a1 = (random.randint(0, 0x10000) for _ in range(2))
+            res = expr.eval(a0, a1)
+            self.assertEqual(res, a0+a1)
+
+    def test_setter_err_check_arg(self):
+        """ Testing __setitem__ errors on invalid argument number"""
+        for argcnt in range(1,10):
+            expr = pyrpn.RPNExpr('0x0', argcnt)
+            for argno in range(argcnt):
+                expr[0] = pyrpn.tokens.Argument(argno)
+            for argno in range(argcnt, 0x100):
+                with self.assertRaises(ValueError):
+                    expr[0] = pyrpn.tokens.Argument(argno)
+
+    def test_setter_err_check_idx(self):
+        """ Testing __setitem__ errors on invalid index"""
+        for elen in range(100):
+            expr = pyrpn.RPNExpr(pyrpn.random_expr(2, elen), 2)
+            for pos in range(len(expr)+1, 0x100):
+                with self.assertRaises(IndexError):
+                    expr[pos] = pyrpn.tokens.Value(0x1)
+
+
+if __name__ == '__main__':
+    unittest.main()
+

tests/tests_rpn_tokens.py

@@ -0,0 +1,162 @@
+#!/usr/bin/python3
+# Copyright 2023 Weber Yann
+#
+# This file is part of rpnifs.
+#
+#        geneifs is free software: you can redistribute it and/or modify
+#        it under the terms of the GNU General Public License as published by
+#        the Free Software Foundation, either version 3 of the License, or
+#        (at your option) any later version.
+#
+#        geneifs is distributed in the hope that it will be useful,
+#        but WITHOUT ANY WARRANTY; without even the implied warranty of
+#        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#        GNU General Public License for more details.
+#
+#        You should have received a copy of the GNU General Public License
+#        along with geneifs.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+import random
+
+import unittest
+
+try:
+    import pyrpn
+except (ImportError, NameError) as e:
+    print("Error importing pyrpn. Try to run make.",
+          file=sys.stderr)
+    raise e
+
+
+class TestRpnToken(unittest.TestCase):
+    """ Testing tokens submodule """
+
+    def test_token_str_convertion(self):
+        """ Testing token <-> str convvertion """
+        expl = ('+', '-', '&',
+                'A1', 'A0', 'A1337',
+                '0x0', '0x1312')
+
+        for estr in expl:
+            t = pyrpn.tokens.Token.from_str(estr)
+            with self.subTest(str=estr, token=t):
+                self.assertEqual(str(t), estr)
+
+class TestRpnTokenOperand(unittest.TestCase):
+    """ Testing operand tokens """
+
+    def test_str_init(self):
+        """ Testing operand token initialization """
+        test_str = ('add', '+', '-', 'sub', 'and', 'xor')
+        for op in test_str:
+            t = pyrpn.tokens.Operand(op)
+            with self.subTest(opstr=op, token=t):
+                self.assertIn(str(t), [t.chr(), t.str()])
+
+    def test_int_init(self):
+        """ Testing operand token initialization with all opcodes """
+        for i in range(pyrpn.tokens.Operand.opcode_max()+1):
+            t = pyrpn.tokens.Operand(i)
+            with self.subTest(opcode=i, token=t):
+                self.assertEqual(i, t.opcode)
+
+    def test_badarg_init(self):
+        """ Testing operand token initialization with bad argument """
+        badargs = ('cxwccseuhefsdfiyg', -1, -150, 
+                   pyrpn.tokens.Operand.opcode_max()+1,
+                   pyrpn.tokens.Operand.opcode_max()*2)
+        badtypes = ('0x12', 'A1',
+                    dict(), tuple(), b'+')
+
+        for badarg in badargs:
+            with self.subTest(badarg=badarg):
+                with self.assertRaises(ValueError):
+                    t = pyrpn.tokens.Operand(badarg)
+
+        for badtype in badtypes:
+            with self.subTest(badtype=badtype):
+                with self.assertRaises(TypeError):
+                    t = pyrpn.tokens.Operand(badtype)
+
+class TestRpnTokenValue(unittest.TestCase):
+    """ Testing value tokens """
+
+    def test_init(self):
+        """ Testing value token initialization """
+        for _ in range(1000):
+            rnd = random.randint(0,(1<<64) -1 )
+            t = pyrpn.tokens.Value(rnd)
+            with self.subTest(token=t, value=rnd):
+                self.assertEqual(int(str(t), 16), rnd)
+
+    def test_badarg_init(self):
+        """ Testing value token initialization with bad argument """
+        badtypes = ('A1', 'add', '+', dict(), tuple(), 5.0)
+        for badtype in badtypes:
+            with self.subTest(badarg=badtype):
+                with self.assertRaises(TypeError):
+                    t = pyrpn.tokens.Value(badtype)
+                    self.assertEqual(int(str(t), 16), badtype)
+
+    def test_badarg_overflow(self):
+        """ Testing value token initialization overflow """
+        badtypes = (1<<64, -1)
+        for badtype in badtypes:
+            with self.subTest(badarg=badtype):
+                with self.assertRaises(OverflowError):
+                    t = pyrpn.tokens.Value(badtype)
+                    self.assertEqual(int(str(t), 16), badtype)
+
+class TestRpnTokenArgument(unittest.TestCase):
+    """ Testing argument tokens """
+
+    def test_init(self):
+        """ Testing argument token initialization """
+        for _ in range(1000):
+            rnd = random.randint(0,(1<<64) -1 )
+            t = pyrpn.tokens.Argument(rnd)
+            with self.subTest(token=t, argument=rnd):
+                self.assertEqual(int(str(t)[1:],10), rnd)
+
+    def test_badarg_init(self):
+        """ Testing argument token initialization with bad argument """
+        badtypes = ('0x1', 'add', '+', dict(), tuple(), 5.0)
+        for badtype in badtypes:
+            with self.subTest(badarg=badtype):
+                with self.assertRaises(TypeError):
+                    t = pyrpn.tokens.Argument(badtype)
+                    self.assertEqual(int(str(t), 16), badtype)
+
+    def test_badarg_overflow(self):
+        """ Testing argument token initialization overflow """
+        badtypes = (1<<64, -1)
+        for badtype in badtypes:
+            with self.subTest(badarg=badtype):
+                with self.assertRaises(OverflowError):
+                    t = pyrpn.tokens.Argument(badtype)
+                    self.assertEqual(int(str(t), 16), badtype)
+
+
+#TODO move in a file dedicated to RPNExpr checks
+class TestTokenExprRepr(unittest.TestCase):
+    """ Test RPNExpr sequence implementation """
+
+    def test_rpnexpr_sequence(self):
+        """ Testing sequence implementation of RPNExpr """
+        for _ in range(100):
+            argc = random.randint(0,100)
+            token_count = random.randint(0, 200)
+            expr_str = pyrpn.random_expr(argc, token_count)
+            expr = pyrpn.RPNExpr(expr_str, argc)
+            with self.subTest(argc=argc, token_count=token_count,
+                              expr=expr, expr_str=expr_str):
+                self.assertEqual(token_count, len(expr))
+                self.assertEqual(' '.join([str(token) for token in expr]),
+                                 expr_str)
+
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/tests_rpniter.py

@@ -0,0 +1,679 @@
+#!/usr/bin/python3
+# copyright 2023 weber yann
+#
+# this file is part of rpnifs.
+#
+#        geneifs is free software: you can redistribute it and/or modify
+#        it under the terms of the gnu general public license as published by
+#        the free software foundation, either version 3 of the license, or
+#        (at your option) any later version.
+#
+#        geneifs is distributed in the hope that it will be useful,
+#        but without any warranty; without even the implied warranty of
+#        merchantability or fitness for a particular purpose.  see the
+#        gnu general public license for more details.
+#
+#        you should have received a copy of the gnu general public license
+#        along with geneifs.  if not, see <http://www.gnu.org/licenses/>.
+#
+
+import copy
+import mmap
+import pickle
+import random
+import sys
+
+import unittest
+
+try:
+    import pyrpn
+except (ImportError, NameError) as e:
+    print("error importing pyrpn. try to run make.",
+          file=sys.stderr)
+    raise e
+
+from utils import *
+
+
+class TestRpnExprCopy(unittest.TestCase):
+    """ Testing RPNExpr sequence method """
+
+
+import unittest
+import warnings
+
+try:
+    import numpy as np
+except (ImportError, NameError) as e:
+    np = None
+    warnings.warn("Numpy not found, some tests skipped", warnings.ImportWarning)
+
+try:
+    import pyrpn
+except (ImportError, NameError) as e:
+    print("Error importing pyrpn. Try to run make.",
+          file=sys.stderr)
+    raise e
+
+def skipIfNoNumpy():
+    if np is not None:
+        return lambda func: func
+    return unittest.skip("Numpy not found")
+
+
+
+class TestRPNIterInit(unittest.TestCase):
+
+    def test_init(self):
+        """ Test instanciation """
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XY,
+                                pyrpn.const.RESULT_COUNT,
+                                (640,480))
+    
+    def test_params_simple(self):
+        """ Test parameters fetch from instanciation """
+        args = [(pyrpn.const.POS_XY,
+                 pyrpn.const.RESULT_COUNT,
+                 (640,480)),
+                (pyrpn.const.POS_LINEAR,
+                 pyrpn.const.RESULT_BOOL,
+                 (1024,))
+        ]
+        for arg in args:
+            with self.subTest(args=arg):
+                rif = pyrpn.RPNIterExpr(*arg)
+                params = rif.get_params()
+                self.assertEqual(params.pos_flag, arg[0])
+                self.assertEqual(params.res_flag, arg[1])
+                self.assertEqual(params.size_lim, arg[2])
+                self.assertIsNone(params.const_values)
+    
+
+    def test_params_const(self):
+        """ Test parameters from instanciation when const values used as result """
+        args = [(pyrpn.const.POS_XY,
+                 pyrpn.const.RESULT_CONST,
+                 (640,480),
+                 (42,)),
+                (pyrpn.const.POS_LINEAR,
+                 pyrpn.const.RESULT_CONST_RGBA,
+                 (1024,), (13,37,13,12))
+        ]
+        for arg in args:
+            with self.subTest(args=arg):
+                rif = pyrpn.RPNIterExpr(*arg)
+                params = rif.get_params()
+                self.assertEqual(params.pos_flag, arg[0])
+                self.assertEqual(params.res_flag, arg[1])
+                self.assertEqual(params.size_lim, arg[2])
+                self.assertEqual(params.const_values, arg[3])
+
+    def test_params_xdim(self):
+        """ Test parameters from instanciation when using xdim positionning """
+        args = [(pyrpn.const.POS_XDIM,
+                 pyrpn.const.RESULT_BOOL,
+                 (2,640,480)),
+                (pyrpn.const.POS_XDIM,
+                 pyrpn.const.RESULT_BOOL,
+                 (5,13,37,13,12,42)),
+        ]
+        for arg in args:
+            with self.subTest(args=arg):
+                rif = pyrpn.RPNIterExpr(*arg)
+                params = rif.get_params()
+                self.assertEqual(params.pos_flag, arg[0])
+                self.assertEqual(params.res_flag, arg[1])
+                self.assertEqual(params.size_lim, arg[2])
+                self.assertIsNone(params.const_values)
+    
+    def test_shape(self):
+        """ Test the shape method """
+        tests = [((pyrpn.const.POS_XY,
+                          pyrpn.const.RESULT_COUNT,
+                          (640,480)), (640,480)),
+                        ((pyrpn.const.POS_LINEAR,
+                          pyrpn.const.RESULT_BOOL,
+                          (1024,)), (1024,)),
+                        ((pyrpn.const.POS_LINEAR,
+                          pyrpn.const.RESULT_RGBA,
+                          (1024,)), (1024,4)),
+                        ((pyrpn.const.POS_XDIM,
+                          pyrpn.const.RESULT_BOOL,
+                          (2,640,480)),  (640,480)),
+                        ((pyrpn.const.POS_XDIM,
+                          pyrpn.const.RESULT_RGB,
+                          (5,13,37,13,12,42)), (13,37,13,12,42,3)),
+                ]
+        for args, expt in tests:
+            with self.subTest(args=args, expt_shape=expt):
+                rif = pyrpn.RPNIterExpr(*args)
+                self.assertEqual(rif.shape(), expt)
+
+    def test_pickling(self):
+        """ Test pickling/unpickling """
+        tests = [((pyrpn.const.POS_XY,
+                          pyrpn.const.RESULT_COUNT,
+                          (640,480)), (640,480)),
+                        ((pyrpn.const.POS_LINEAR,
+                          pyrpn.const.RESULT_BOOL,
+                          (1024,)), (1024,)),
+                        ((pyrpn.const.POS_LINEAR,
+                          pyrpn.const.RESULT_RGBA,
+                          (1024,)), (1024,4)),
+                        ((pyrpn.const.POS_XDIM,
+                          pyrpn.const.RESULT_BOOL,
+                          (2,640,480)),  (640,480)),
+                        ((pyrpn.const.POS_XDIM,
+                          pyrpn.const.RESULT_RGB,
+                          (5,13,37,13,12,42)), (13,37,13,12,42,3)),
+                ]
+        for args, expt in tests:
+            for _ in range(10):
+                rif = pyrpn.RPNIterExpr(*args)
+                for ex in rif.expressions:
+                    ex.mutate(n_mutations = 15)
+                with self.subTest(rif=rif):
+                    st = pickle.dumps(rif)
+                    rif2 = pickle.loads(st)
+                    st2 = pickle.dumps(rif2)
+                    self.assertEqual(st, st2)
+
+    @skipIfNoNumpy()
+    def test_buffer(self):
+        """ Test the len on buffer interface using numpy """
+        args = [((pyrpn.const.POS_XY,
+                  pyrpn.const.RESULT_COUNT,
+                  (640,480)), 640*480),
+                ((pyrpn.const.POS_LINEAR,
+                  pyrpn.const.RESULT_BOOL,
+                  (1024,)), 1024),
+                ((pyrpn.const.POS_XDIM,
+                  pyrpn.const.RESULT_BOOL,
+                  (2,640,480)),  640*480),
+                ((pyrpn.const.POS_XDIM,
+                  pyrpn.const.RESULT_BOOL,
+                  (5,13,37,13,12,42)), 13*37*13*12*42),
+        ]
+        
+        for arg, expt_sz in args:
+                with self.subTest(args=arg):
+                    rif = pyrpn.RPNIterExpr(*arg)
+                    arr = np.frombuffer(rif, dtype=np.uint64)
+                    self.assertEqual(len(arr), expt_sz)
+                    arr += 0x11111111 # check writing to all bytes
+
+    @skipIfNoNumpy()
+    def test_mmap_accessor(self):
+        """ Testing mmap access """
+        args = (pyrpn.const.POS_XY,
+                  pyrpn.const.RESULT_COUNT,
+                  (640,480))
+        rif = pyrpn.RPNIterExpr(*args)
+        arr = np.frombuffer(rif, dtype=np.uint64)
+        arr[42] = 1312
+        arr2 = np.frombuffer(rif.mmap, dtype=np.uint64)
+        self.assertTrue((arr == arr2).all())
+
+        rif.mmap.write(b'hello world !')
+        arr = np.frombuffer(rif, dtype=np.uint64)
+        arr2 = np.frombuffer(rif.mmap, dtype=np.uint64)
+        self.assertTrue((arr == arr2).all())
+
+    @skipIfNoNumpy()
+    def test_mmap_argument(self):
+        """ Testing mmap argument """
+        args = (pyrpn.const.POS_XY,
+                  pyrpn.const.RESULT_COUNT,
+                  (640,480))
+        params = pyrpn.RPNIterExpr.params(*args)
+        mm = mmap.mmap(-1, params.memory_size)
+        mm.write(b'Hello world !')
+        rif = pyrpn.RPNIterExpr(*args, mmap=mm)
+        self.assertEqual(mm, rif.mmap)
+        arr = np.frombuffer(rif, dtype=np.uint64)
+        arr2 = np.frombuffer(mm, dtype=np.uint64)
+        self.assertTrue((arr == arr2).all())
+
+    @skipIfNoNumpy()
+    def test_mmap_update(self):
+        """ Testing mmap update """
+        args = (pyrpn.const.POS_XY, pyrpn.const.RESULT_RGB,
+                (640,480))
+        params = pyrpn.RPNIterExpr.params(*args)
+        mm1 = mmap.mmap(-1, params.memory_size)
+        arr = np.frombuffer(mm1, dtype=np.uint64)
+        for _ in range(1024):
+            idx = random.randint(0, params.memory_size // 8)
+            val = random.randint(0, 0xFFFFFFFF)
+            arr[idx] = val
+
+        rif = pyrpn.RPNIterExpr(*args)
+        pos = 0
+        while pos == 0:
+            for exp in rif.expressions:
+                exp.mutate(n_mutations=15)
+            pos = rif.step(pos)
+
+        for _ in range(4096):
+            pos = rif.step(pos)
+
+        arr2 = arrorig = np.frombuffer(rif, dtype=np.uint64)
+        self.assertFalse((arr == arr2).all())
+
+        rif.set_mmap(mm1)
+        arr2 = np.frombuffer(rif, dtype=np.uint64)
+        self.assertTrue((arr == arr2).all())
+        del(arr)
+        del(arr2)
+
+        mm1.write(b'Hello world !')
+        arr2 = np.frombuffer(rif, dtype=np.uint64)
+        arr = np.frombuffer(mm1, dtype=np.uint64)
+        self.assertTrue((arr == arr2).all())
+
+        arr = np.frombuffer(mm1, dtype=np.uint64)
+        self.assertFalse((arr == arrorig).all())
+
+    def test_str(self):
+        """ Test string representation of rif """
+
+        tests = [[pyrpn.const.POS_XY, pyrpn.const.RESULT_RGB, (1024,1024)],
+                 [pyrpn.const.POS_LINEAR, pyrpn.const.RESULT_CONST_RGBA, (1024,), (255,0,0,255)],
+                 [pyrpn.const.POS_XY, pyrpn.const.RESULT_CONST_RGBA, (640,480), (255,0,0,255)],
+                 [pyrpn.const.POS_LINEAR, pyrpn.const.RESULT_RGBA, (1024,)],
+                 [pyrpn.const.POS_XDIM, pyrpn.const.RESULT_RGB, (4, 2, 2, 640, 480)],
+                 [pyrpn.const.POS_XY, pyrpn.const.RESULT_CONST, (1024,512),
+                     (2,)],
+        ]
+        for args in tests:
+            rif = pyrpn.RPNIterExpr(*args)
+            [expr.mutate(n_mutations=5) for expr in rif]
+            codes = {spl[1]:spl[2].strip('"')
+                     for spl in [s.split('=') for s in str(rif).split(';')
+                                 if len(s.strip())]}
+            argc = rif.get_params().argc
+            for key, orig in rif.items():
+                expr = pyrpn.RPNExpr(codes[key], argc)
+                with self.subTest(rif=rif, key=key, orig=orig, clone=expr):
+                    self.assertEqual(orig, expr)
+            str_repr = repr(rif)
+
+
+class TestRPNIterCoordinates(unittest.TestCase):
+    """ Testing methods for coordinates <-> position convertions """
+
+    def test_position_linear(self):
+        """ Testing linear coordinate convertion methods """
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_LINEAR,
+                    pyrpn.const.RESULT_CONST,
+                    (256,), (42,))
+
+        for pos in range(256):
+            with self.subTest(rif=rif, position=pos, expt=(pos,)):
+                res = rif.from_pos(pos)
+                self.assertEqual(res, (pos,))
+            with self.subTest(rif=rif, expt=pos, coord=(pos,)):
+                res = rif.to_pos(pos)
+                self.assertEqual(res, pos)
+
+    def test_position_xy(self):
+        """ Testing XY coordinate convertion methods """
+        for _ in Progress(5):
+            sz = (random.randint(32,128), random.randint(32,128))
+            rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XY,
+                    pyrpn.const.RESULT_CONST,
+                    sz, (42,))
+            for pos in range(sz[0]*sz[1]):
+                coord = rif.from_pos(pos)
+                expt = (pos % sz[0], pos // sz[0])
+                with self.subTest(sz=sz, pos=pos, expt=expt, result=coord):
+                    self.assertEqual(expt, coord)
+                result = rif.to_pos(*coord)
+                with self.subTest(sz=sz, coord=coord, expt=pos, result=result):
+                    self.assertEqual(result, pos)
+
+    def test_position_xy_neg(self):
+        """ Testing XY coordinates with negative values """
+        sz = (100,200)
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XY,
+                pyrpn.const.RESULT_CONST,
+                sz, (42,))
+        
+        for _ in range(1000):
+            ny = random.randint(-0x1000000, sz[1])
+            nx = random.randint(0,sz[0]-1)
+            pos = rif.to_pos(nx, ny)
+            coord = rif.from_pos(pos)
+            self.assertEqual(coord, (nx, (ny%(1<<64))%sz[1]))
+
+    def test_position_xy_random(self):
+        """ Testing XY coordinate convertion methods (random samples)"""
+        for _ in Progress(256):
+            sz = (random.randint(32,4096), random.randint(32,4096))
+            rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XY,
+                    pyrpn.const.RESULT_CONST,
+                    sz, (42,))
+            for _ in range(500):
+                pos = random.randint(0, (sz[0]*sz[1])-1)
+                coord = rif.from_pos(pos)
+                expt = (pos % sz[0], pos // sz[0])
+                with self.subTest(sz=sz, pos=pos, expt=expt, result=coord):
+                    self.assertEqual(expt, coord)
+                result = rif.to_pos(*coord)
+                with self.subTest(sz=sz, coord=coord, expt=pos, result=result):
+                    self.assertEqual(result, pos)
+
+    def test_position_xdim(self):
+        """ Testing XDIM coordinate convertion methods """
+        ndim = 5
+        resol = [8 for _ in range(ndim)]
+        sz = [ndim]+resol
+        linear_size = 1
+        for e in resol:
+            linear_size *= e
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XDIM,
+                pyrpn.const.RESULT_CONST,
+                sz, (42,))
+        for pos in range(linear_size):
+            coord = rif.from_pos(pos)
+            expt = []
+            cur = pos
+            for dim in resol:
+                elt = cur % dim
+                cur //= dim
+                expt.append(elt)
+            expt = tuple(expt)
+            with self.subTest(sz=sz, pos=pos, expt=expt, result=coord):
+                self.assertEqual(expt, coord)
+            result = rif.to_pos(*coord)
+            with self.subTest(sz=sz, coord=coord, expt=pos, result=result):
+                self.assertEqual(result, pos)
+
+    def test_position_xdim_neg(self):
+        """ Testing XDIM negative position convertion """
+        # TODO test negativ for other coordinate systems
+        ndim = 4
+        szlim = (ndim, 10,20,30,40)
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XDIM,
+                pyrpn.const.RESULT_CONST,
+                szlim, (42,))
+
+        pos = rif.to_pos(0,0,-5,0)
+        self.assertEqual(rif.from_pos(pos), (0,0,(-5%(1<<64))%30, 0))
+
+    def test_position_xdim(self):
+        """ Testing XDIM coordinate convertion methods (random sampling)"""
+        for _ in Progress(16):
+            ndim = random.randint(3,8)
+            resol = [random.randint(2, 16) for _ in range(ndim)]
+            sz = [ndim]+resol
+            linear_size = 1
+            for e in resol:
+                linear_size *= e
+            rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XDIM,
+                    pyrpn.const.RESULT_CONST,
+                    sz, (42,))
+            for _ in range(2000):
+                pos = random.randint(0, linear_size-1)
+                coord = rif.from_pos(pos)
+                expt = []
+                cur = pos
+                for dim in resol:
+                    elt = cur % dim
+                    cur //= dim
+                    expt.append(elt)
+                expt = tuple(expt)
+                with self.subTest(sz=sz, pos=pos, expt=expt, result=coord):
+                    self.assertEqual(expt, coord)
+                result = rif.to_pos(*coord)
+                with self.subTest(sz=sz, coord=coord, expt=pos, result=result):
+                    self.assertEqual(result, pos)
+
+
+class TestRPNIterConst(unittest.TestCase):
+    """ Testing various coordinate systems with constant result value """
+
+    @skipIfNoNumpy()
+    def test_simple_linear_const(self):
+        """ Testing a simple constant result on linear coord  """
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_LINEAR,
+                                pyrpn.const.RESULT_CONST,
+                                (512,), (42,))
+
+        data = np.frombuffer(rif, dtype=np.uint64)
+        for elt in data:
+            self.assertEqual(elt, 0)
+
+        rif['X'] = 'A0 A1 +'
+        pos = rif.step(0)
+        self.assertEqual(pos, 0)
+        self.assertEqual(data[0], 42)
+
+        data = np.frombuffer(rif, dtype=np.uint64)
+        self.assertEqual(data[0], 42)
+        for elt in data[1:]:
+            self.assertEqual(elt, 0)
+
+
+
+        pos = rif.step(0)
+        self.assertEqual(pos, 42)
+
+        data = np.frombuffer(rif, dtype=np.uint64)
+        for i, elt in enumerate(data):
+            with self.subTest(cur_pos=i):
+                if i in (0, 42):
+                    self.assertEqual(elt, 42)
+                else:
+                    self.assertEqual(elt, 0)
+
+
+        pos = rif.step(1)
+        self.assertEqual(pos, 1)
+        pos = rif.step(pos)
+        self.assertEqual(pos, 43)
+
+        newpos = 512 - 40
+        pos = rif.step(newpos)
+        self.assertEqual(pos, newpos)
+        pos = rif.step(pos)
+        self.assertEqual(pos, 2)
+
+    def test_random_linear_const(self):
+        """ Testing linear coord with const with random expressions """
+        for _ in range(200):
+            const_val = random.randint(0,0xFFFF)
+            rif = pyrpn.RPNIterExpr(pyrpn.const.POS_LINEAR,
+                                    pyrpn.const.RESULT_CONST,
+                                    (512,), (const_val,))
+            rif['X'].mutate(n_mutations=random.randint(10,100))
+            pos=0
+            all_pos=[]
+            for  _ in range(100):
+                pos = rif.step(pos)
+                all_pos.append(pos)
+                data = np.frombuffer(rif, dtype=np.uint64)
+                self.assertEqual(data[pos], const_val)
+            for i, elt in enumerate(np.frombuffer(rif, dtype=np.uint64)):
+                if i in all_pos:
+                    self.assertEqual(elt, const_val)
+                else:
+                    self.assertEqual(elt, 0)
+
+    def test_simple_xy_const(self):
+        """ Testing xy coord with const value """
+        sz = (1024,256)
+
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XY,
+                                pyrpn.const.RESULT_CONST,
+                                sz, (42,))
+        
+        rif['X'] = 'A0 A1 +'
+        rif['Y'] = 'A2'
+
+        pos = rif.step(0)
+        self.assertEqual(pos, 0)
+
+        pos = rif.step(0)
+        self.assertEqual(rif.from_pos(pos), (0,42))
+
+        pos = rif.step(rif.to_pos(1,1))
+        self.assertEqual(rif.from_pos(pos), (2,0))
+
+        pos = rif.step(rif.to_pos(1,1))
+        self.assertEqual(rif.from_pos(pos), (2,0))
+
+        pos = rif.step(rif.to_pos(2,0))
+        self.assertEqual(rif.from_pos(pos), (2,42))
+
+    def test_random_xy_const(self):
+        """ Testing xy coord with const with random expressions """
+        for _ in Progress(200):
+            const_val = random.randint(0,0xFFFF)
+            rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XY,
+                                    pyrpn.const.RESULT_CONST,
+                                    (1024,1024,), (const_val,))
+            rif['X'].mutate(n_mutations=random.randint(10,100))
+            rif['Y'].mutate(n_mutations=random.randint(10,100))
+            pos=0
+            all_pos=[]
+            for  _ in range(100):
+                sys.stdout.flush()
+                pos = rif.step(pos)
+                all_pos.append(pos)
+                data = np.frombuffer(rif, dtype=np.uint64)
+                self.assertEqual(data[pos], const_val)
+            expt = np.zeros(len(data), dtype=np.uint64)
+            for p in all_pos:
+                expt[p] = const_val
+            self.assertTrue((expt == data).all())
+
+    def test_simple_xdim_const(self):
+        """ Testing xdim coord with const """
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XDIM,
+                                pyrpn.const.RESULT_CONST,
+                                (4,100,200,300,400), (42,))
+        rif['D0'] = 'A1'
+        rif['D1'] = 'A0 A2 +'
+        rif['D2'] = 'A0 A1 -'
+        rif['D3'] = 'A3 A4 +'
+
+        pos = rif.step(0)
+        self.assertEqual(pos, 0)
+        pos = rif.step(0)
+        self.assertEqual(rif.from_pos(pos), (0,0,0,42))
+
+        pos = rif.step(5)
+        self.assertEqual(rif.from_pos(pos), (0,5,5,0))
+        pos = rif.step(rif.to_pos(0,5,5,0))
+        self.assertEqual(rif.from_pos(pos), (5,5,(-5%(1<<64))%300,42))
+        expt = (5,5,-5,42)
+        expt_pos = rif.to_pos(*expt)
+        self.assertEqual(pos, expt_pos)
+
+    def test_linear_rgb(self):
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_LINEAR,
+                                pyrpn.const.RESULT_RGB,
+                                (512,))
+        rif['X'] = 'A0 2 +'
+        rif['R'] = 'A0 1 +'
+        rif['G'] = 'A1 A2 + 255 %'
+        rif['B'] = 'A2 2 * A3 +'
+        
+        pos = rif.step(0)
+        self.assertEqual(pos, 2)
+        data = np.frombuffer(rif, dtype=np.uint64).reshape(rif.shape())
+        colors = list(data[2])
+        self.assertEqual(colors, [1,0,0])
+
+        pos = rif.step(2)
+        self.assertEqual(pos, 4)
+        data = np.frombuffer(rif, dtype=np.uint64).reshape(rif.shape())
+        colors = list(data[4])
+        self.assertEqual(colors, [3, 1, 0])
+
+    def test_linear_rgba(self):
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_LINEAR,
+                                pyrpn.const.RESULT_RGBA,
+                                (512,))
+        rif['X'] = 'A0 2 +'
+        rif['R'] = 'A0 1 +'
+        rif['G'] = 'A1 A2 + 255 %'
+        rif['B'] = 'A2 2 * A3 +'
+        rif['A'] = 'A1 A2 A3 + +'
+        
+        pos = rif.step(0)
+        self.assertEqual(pos, 2)
+        data = np.frombuffer(rif, dtype=np.uint64).reshape(rif.shape())
+        colors = list(data[2])
+        self.assertEqual(colors, [1,0,0,0])
+
+        pos = rif.step(2)
+        self.assertEqual(pos, 4)
+        data = np.frombuffer(rif, dtype=np.uint64).reshape(rif.shape())
+        colors = list(data[4])
+        self.assertEqual(colors, [3, 1, 0, 1])
+
+        pos = rif.step(4)
+        self.assertEqual(pos, 6)
+        data = np.frombuffer(rif, dtype=np.uint64).reshape(rif.shape())
+        colors = list(data[6])
+        self.assertEqual(colors, [5, 4, 2, 4])
+
+        pos = rif.step(6)
+        self.assertEqual(pos, 8)
+        data = np.frombuffer(rif, dtype=np.uint64).reshape(rif.shape())
+        colors = list(data[8])
+        self.assertEqual(colors, [7, 9, 10, 11])
+
+
+class TestRpnIterMutate(unittest.TestCase):
+    """ @todo Complete tests
+    """
+
+    def test_mutate(self):
+        """ Testing default mutation parameters """
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_LINEAR,
+                                pyrpn.const.RESULT_RGBA,
+                                (512,))
+
+        rif.mutate()
+        rif.mutate(10)
+
+    def test_mutate_weights(self):
+        """ Testing mutation expression weights """
+        params = [1,1,0,0,0,(0,0,0),(0,0,0)]
+        param = pyrpn.RPNMutationParamsTuple(params)
+        for n_mut in range(0,100,10):
+            for i in range(4):
+                rif = pyrpn.RPNIterExpr(pyrpn.const.POS_LINEAR,
+                                        pyrpn.const.RESULT_RGB,
+                                        (512,))
+                weights = [1 if i == j else 0 for j in range(4)]
+                rif.mutate(n_mut, params=param, weights=weights)
+                rpn_exprs = list(rif.values())
+                for j in range(4):
+                    with self.subTest(params=param, weights=weights, expr=j):
+                        if j == i:
+                            self.assertEqual(len(rpn_exprs[j]),  n_mut)
+                        else:
+                            self.assertEqual(len(rpn_exprs[j]), 0)
+        n_mut = 400
+        for w in Progress(range(100)):
+            weights = [w for _ in range(4)]
+            rif = pyrpn.RPNIterExpr(pyrpn.const.POS_LINEAR,
+                                    pyrpn.const.RESULT_RGB,
+                                    (512, ))
+            
+            rif.mutate(n_mut, params=param, weights=weights)
+            rpn_exprs = list(rif.values())
+            for rpnexpr in rpn_exprs:
+                expr_len = len(rpnexpr)
+                expt_len = n_mut / 4
+                score = abs(1 - (expt_len / expr_len))
+                self.assertLess(score, 0.5)
+
+
+if __name__ == '__main__':
+    unittest.main()
+

tests/tests_rpniter_seq.py

@@ -0,0 +1,100 @@
+#!/usr/bin/python3
+# copyright 2023 weber yann
+#
+# this file is part of rpnifs.
+#
+#        geneifs is free software: you can redistribute it and/or modify
+#        it under the terms of the gnu general public license as published by
+#        the free software foundation, either version 3 of the license, or
+#        (at your option) any later version.
+#
+#        geneifs is distributed in the hope that it will be useful,
+#        but without any warranty; without even the implied warranty of
+#        merchantability or fitness for a particular purpose.  see the
+#        gnu general public license for more details.
+#
+#        you should have received a copy of the gnu general public license
+#        along with geneifs.  if not, see <http://www.gnu.org/licenses/>.
+#
+
+import copy
+import pickle
+import random
+import sys
+
+import unittest
+
+try:
+    import pyrpn
+except (ImportError, NameError) as e:
+    print("error importing pyrpn. try to run make.",
+          file=sys.stderr)
+    raise e
+
+
+class TestRpnIterSequence(unittest.TestCase):
+    """ Testing RPNIterExpr sequence methods """
+
+    def test_len(self):
+        """ Testing RPNIterExpr.__len__ with various values/constants """
+        tests = [([pyrpn.const.POS_XY, pyrpn.const.RESULT_RGB, (1024,1024)],
+                   5),
+                 ([pyrpn.const.POS_LINEAR, pyrpn.const.RESULT_CONST_RGBA, (1024,), (255,0,0,255)],
+                   1),
+                 ([pyrpn.const.POS_XY, pyrpn.const.RESULT_CONST_RGBA, (640,480), (255,0,0,255)],
+                   2),
+                 ([pyrpn.const.POS_LINEAR, pyrpn.const.RESULT_RGBA, (1024,)],
+                   5),
+                 ([pyrpn.const.POS_XDIM, pyrpn.const.RESULT_RGB, (4, 2, 2, 640, 480)],
+                  7),
+        ]
+        for args, expt_len in tests:
+            rif=pyrpn.RPNIterExpr(*args)
+            with self.subTest(rif=rif, expt_len=expt_len, args=args):
+                self.assertEqual(expt_len, len(rif))
+
+
+
+class TestRPNIterMapping(unittest.TestCase):
+    """ Testing RPNIterExpr mapping methods """
+
+
+    def test_keys(self):
+        """ Testing RPNIterExpr.__getitem__ with various values/constants """
+        tests = [([pyrpn.const.POS_XY, pyrpn.const.RESULT_RGB, (1024,1024)],
+                  ['X', 'Y', 'R', 'G', 'B']),
+                 ([pyrpn.const.POS_LINEAR, pyrpn.const.RESULT_CONST_RGBA, (1024,), (255,0,0,255)],
+                   ['X']),
+                 ([pyrpn.const.POS_XY, pyrpn.const.RESULT_CONST_RGBA, (640,480), (255,0,0,255)],
+                   ['X', 'Y']),
+                 ([pyrpn.const.POS_LINEAR, pyrpn.const.RESULT_RGBA, (1024,)],
+                   ['X', 'R', 'G', 'B', 'A']),
+                 ([pyrpn.const.POS_XDIM, pyrpn.const.RESULT_RGB, (4, 2, 2, 640, 480)],
+                     ['D0', 'D1', 'D2', 'D3', 'R', 'G', 'B']),
+        ]
+        for args, keys in tests:
+            rif=pyrpn.RPNIterExpr(*args)
+            for curkey in keys:
+                with self.subTest(rif=rif, key=curkey):
+                    expr = rif[curkey]
+            expt = set(keys)
+            self.assertEqual(set(rif.keys()), expt)
+            for key, value in rif.items():
+                with self.subTest(item=(key, value), id1=id(rif[key]),
+                        id2=id(value), rif=rif):
+                    self.assertEqual(rif[key], value)
+
+    def test_assignement(self):
+        """ Testing RPNIterExpr.__setitem__ """
+        rif = pyrpn.RPNIterExpr(pyrpn.const.POS_XY, pyrpn.const.RESULT_RGB, (1024,1024))
+        rif['X'] = 'A0 A1 +'
+        for _ in range(1000):
+            a, b = [random.randint(0,0xFFFF) for _  in range(2)]
+            args = (a,b, 0,0,0)
+            self.assertEqual(a+b, rif['X'].eval(*args))
+            self.assertEqual(0, rif['Y'].eval(*args))
+
+
+if __name__ == '__main__':
+    unittest.main()
+

tests/tests_tokens.py

@@ -0,0 +1,92 @@
+#!/usr/bin/python3
+# copyright 2023 weber yann
+#
+# this file is part of rpnifs.
+#
+#        geneifs is free software: you can redistribute it and/or modify
+#        it under the terms of the gnu general public license as published by
+#        the free software foundation, either version 3 of the license, or
+#        (at your option) any later version.
+#
+#        geneifs is distributed in the hope that it will be useful,
+#        but without any warranty; without even the implied warranty of
+#        merchantability or fitness for a particular purpose.  see the
+#        gnu general public license for more details.
+#
+#        you should have received a copy of the gnu general public license
+#        along with geneifs.  if not, see <http://www.gnu.org/licenses/>.
+#
+import random
+import sys
+
+import unittest
+
+try:
+    import pyrpn
+except (ImportError, NameError) as e:
+    print("error importing pyrpn. try to run make.",
+          file=sys.stderr)
+    raise e
+
+
+class TestTokens(unittest.TestCase):
+    """ Testing tokens module """
+
+    def test_argument(self):
+        """ Testing tokens.Argument type  init """
+        for argno in range(1024):
+            arg = pyrpn.tokens.Argument(argno)
+            with self.subTest(argno=argno, arg=arg):
+                self.assertEqual(arg.argno, argno)
+
+    def test_value(self):
+        """ Testing tokens.Value type init """
+        for _ in range(1024):
+            v = random.randint(0, 0xFFFFFFFF)
+            val = pyrpn.tokens.Value(v)
+            with self.subTest(val=v, token_value=val):
+                self.assertEqual(val.value, v)
+
+    def test_op(self):
+        """ Testing tokens.Operand type init """
+        for opcode in range(pyrpn.tokens.Operand.opcode_max()+1):
+            operand = pyrpn.tokens.Operand(opcode)
+            with self.subTest(opcode=opcode, operand=operand):
+                self.assertEqual(opcode, operand.opcode)
+
+            op_fromstr = pyrpn.tokens.Operand(str(operand))
+            with self.subTest(from_str=op_fromstr, opcode=opcode):
+                self.assertEqual(opcode, op_fromstr.opcode)
+
+    def test_cmp(self):
+        """ Testing token comparison """
+        tests = {
+                'lt': ([('A0', 'A1'), ('0x10', '0x11'), ('-', '/'),
+                        ('-', 'A0'), ('-', '0x1'), ('A0', '0x1')],
+                       [('A1', 'A0'), ('0x11', '0x10'), ('/', '-'),
+                        ('A0', '-'), ('0x1000', '+'), ('0x0', 'A1')]),
+                'eq': [[('A0', 'A0'), ('0x1', '0x1'), ('+', '+')],
+                       []]
+        }
+        tests['gt'] = (tests['lt'][1], tests['lt'][0])
+        tests['le'] = [tests['lt'][i] + tests['eq'][i] for i in range(2)]
+        tests['ge'] = [tests['gt'][i] + tests['eq'][i] for i in range(2)]
+        tests['eq'][1] = [('A0', '0x0'), ('A0', '+'), ('0x0', '+')] 
+
+        assertions = {'lt': (self.assertLess, self.assertGreaterEqual),
+                'le': (self.assertLessEqual, self.assertGreater),
+                'eq': (self.assertEqual, self.assertNotEqual),
+                'gt': (self.assertGreater, self.assertLessEqual),
+                'ge': (self.assertGreaterEqual, self.assertLess)}
+
+        for comparison, (cmp_true, cmp_false) in tests.items():
+            assert_true, assert_false = assertions[comparison]
+            lst = [(assert_true, cmp_true), (assert_false, cmp_false)]
+            for assertion, cmplist in lst:
+                for test in cmplist:
+                    a=pyrpn.tokens.Token.from_str(test[0])
+                    b=pyrpn.tokens.Token.from_str(test[1])
+                    with self.subTest(assertion=assertion, a=a, b=b):
+                        assertion(a, b)
+
+

tests/utils.py

@@ -0,0 +1,204 @@
+#!/usr/bin/python3
+# copyright 2023 weber yann
+#
+# this file is part of rpnifs.
+#
+#        geneifs is free software: you can redistribute it and/or modify
+#        it under the terms of the gnu general public license as published by
+#        the free software foundation, either version 3 of the license, or
+#        (at your option) any later version.
+#
+#        geneifs is distributed in the hope that it will be useful,
+#        but without any warranty; without even the implied warranty of
+#        merchantability or fitness for a particular purpose.  see the
+#        gnu general public license for more details.
+#
+#        you should have received a copy of the gnu general public license
+#        along with geneifs.  if not, see <http://www.gnu.org/licenses/>.
+#
+
+from ctypes import *
+import struct
+import sys
+import math
+
+try:
+    import pyrpn
+except (ImportError, NameError) as e:
+    print("error importing pyrpn. try to run make.",
+          file=sys.stderr)
+    raise e
+
+
+VERBOSE = '-v' in sys.argv or '--verbose' in sys.argv
+
+class Progress:
+
+    STYLE = '|/-\\'
+
+    def __init__(self, iterator, count=None):
+        if isinstance(iterator, int):
+            self.__iter = range(iterator)
+            self.__count = iterator
+        else:
+            self.__iter = iterator
+            try:
+                self.__count = len(iterator) if count is None else count
+            except Exception:
+                self.__count = None
+        self.__iter = iter(self.__iter)
+
+        self.__cur = 0
+        if self.__count is not None:
+            count_sz = math.floor(math.log10(self.__count)+1)
+            self.__fmt = '%%%dd/%%%dd' % (count_sz, count_sz)
+    
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return self.next()
+
+    def next(self):
+        if not VERBOSE or self.__count is None:
+            clean_len = 1
+            if VERBOSE and self.__count == 0:
+                sys.stderr.write('\b')
+            sys.stderr.write(self.STYLE[self.__cur%len(self.STYLE)])
+            sys.stderr.write('\b')
+        else:
+            msg = self.__fmt % (self.__cur+1, self.__count)
+            clean_len = len(msg)
+            msg += '\b' * clean_len
+            sys.stderr.write(msg)
+        sys.stderr.flush()
+        self.__cur += 1
+        try:
+            return next(self.__iter)
+        except Exception as expt:
+            sys.stderr.write(' '*clean_len + '\b'  *  clean_len)
+            raise expt
+
+
+def RPNExpr_pickle_bin_dump(state):
+    """ Return a dict containing rpn state dump fields
+    """
+    ser_fields = ['token_sz', 'stack_sz', 'argc', 'state']
+    ser_expr = struct.unpack('QQQQ', state[:32])
+    ser_expr = dict(zip(ser_fields, ser_expr))
+    err = state[32:32+128]
+    if err == b'\x00' * 128:
+        err = None
+    ser_expr['err_reason'] = err
+
+    token_sz = 24
+    tok_start = 32+128
+    tok_end = tok_start + (token_sz * ser_expr['token_sz'])
+
+    tokens = [struct.unpack('QQQ', state[tok_start+(24*i):tok_start+(24*(i+1))])
+              for i in range(ser_expr['token_sz'])]
+
+    tok_types = ['op', 'arg', 'val']
+    #pretty_toks=[{'type': tok_types[tok[0]],
+    #              'type_raw': tok[0],
+    #              'v1': hex(tok[1]),
+    #              'v2': hex(tok[2])}
+    #             for tok in tokens]
+    pretty_toks = [(tok_types[tok[0]],)+tok for tok in tokens]
+    
+    ser_expr['tokens'] = pretty_toks
+    #ser_expr['tokens'] = tokens
+
+    stack_start = tok_end
+    stack_end = stack_start + (8*ser_expr['stack_sz'])
+    if len(state) < stack_end:
+        print("ERROR expt size = %d but got %d" % (stack_end, len(state)))
+        ser_expr['stack'] = 'FAULT'
+        return ser_expr
+    stack = struct.unpack('L'*ser_expr['stack_sz'], state[stack_start:stack_end])
+    ser_expr['stack'] = stack
+
+    return ser_expr
+
+
+
+def RPNIterExpr_pickle_bin_dump(dump):
+
+    import pprint
+
+    params_fields = ('mem_sz','value_sz','rpn_sz','rpn_argc','rpn_stack_sz',
+            'getarg_f','getres_f','data_ptr')
+    data_fields = ('pos_flag','res_flag', 'size_lim_ptr', 'ndim', 'const_val_ptr')
+
+    ptr = dump[0:8]
+    params = dump[8:72]
+    data = dump[72:104]
+
+    print("PARAMS : %r" % params)
+    print("DATA   : %r" % data)
+
+    params = struct.unpack('LLLLbPPP', params)
+    params = dict(zip(params_fields, params))
+
+    data = struct.unpack('hhPLP', data)
+    data = dict(zip(data_fields, data))
+
+    params['data'] = data
+
+    sz_sz = data['ndim'] + (1 if data['pos_flag'] == pyrpn.const.POS_XDIM else 0)
+    sz_end = 104 + 8*sz_sz
+    sz_lim = struct.unpack('L' * sz_sz, dump[104:sz_end])
+    data['sz_lim'] = sz_lim
+
+    const_sz = 0
+    if data['res_flag'] == pyrpn.const.RESULT_CONST:
+        const_sz = 1
+    elif data['res_flag'] == pyrpn.const.RESULT_CONST_RGBA:
+        const_sz = 4
+
+    const_end = sz_end + (8*const_sz)
+
+    if const_sz:
+        const_val = struct.unpack('L'*const_sz, dump[sz_end:const_end])
+    else:
+        const_val = []
+
+
+    print("SZLIM  : %r" % (dump[104:sz_end]))
+    print("CONST  : %r" % (dump[sz_end:const_end]))
+
+    data['const_val'] = const_val
+
+    rpn_sz_end = const_end + (8*params['rpn_sz'])
+
+
+    rpn_sz = struct.unpack('L'*params['rpn_sz'], dump[const_end:rpn_sz_end])
+    
+    ddump = {'params': params, 'rpn_sz': rpn_sz, 'expr': dict()}
+
+    rpn_buf_start = rpn_sz_end
+    for i in range(params['rpn_sz']):
+        rpn_buf_end = rpn_buf_start + rpn_sz[i]
+        rpn_state = dump[rpn_buf_start:rpn_buf_end]
+        print("RPN#%d[%d:%d] = %r" % (i, rpn_buf_start, rpn_buf_end, rpn_state))
+        try:
+            ddump['expr'][i] = RPNExpr_pickle_bin_dump(rpn_state)
+        except Exception as expt:
+            print("ERROR : ", expt)
+        rpn_buf_start = rpn_buf_end
+
+    return ddump
+
+
+def bindump(data):
+
+    for i in range(0,len(data), 0x10):
+        print("%02X " % i, data[i:i+0x10])
+    if len(data) % 0x10:
+        last = ((len(data) / 0x10)+1)
+        print("%02X " % last, data[last:])
+
+    
+
+
+