import sys, os, re, pdb, StringIO DEBUG = 0 def log(msg): if DEBUG>0: sys.stderr.write(msg+"\n") def escape_for_string(string): result = string result = result.encode("string-escape") result = result.replace('"',r'\"') return result class Module: def __init__(self, name): self.name = name self.constants = set() self.classes = {} self.headers = '#include \n' self.files = [] def __str__(self): result = "Module %s\n" % (self.name) l = self.classes.values() l.sort() for attr in l: if attr.is_active(): result += " %s\n" % attr l = list(self.constants) l.sort() result += 'Constants:\n' for attr, type in l: result += " %s\n" % attr return result init_string = '' def initialization(self): result = self.init_string + """ talloc_set_log_fn((void(*)(const char*))printf); //talloc_enable_leak_report(); """ for cls in self.classes.values(): if cls.is_active(): result += cls.initialise() return result def add_constant(self, constant, type="numeric"): """ This will be called to add #define constant macros """ self.constants.add((constant, type)) def add_class(self, cls, handler): self.classes[cls.class_name] = cls ## Make a wrapper in the type dispatcher so we can handle ## passing this class from/to python type_dispatcher[cls.class_name] = handler def private_functions(self): """ Emits hard coded private functions for doing various things """ return """ /* The following is a static array mapping CLASS() pointers to their python wrappers. This is used to allow the correct wrapper to be chosen depending on the object type found - regardless of the prototype. This is basically a safer way for us to cast the correct python type depending on context rather than assuming a type based on the .h definition. For example consider the function AFFObject Resolver.open(uri, mode) The .h file implies that an AFFObject object is returned, but this is not true as most of the time an object of a derived class will be returned. In C we cast the returned value to the correct type. In the python wrapper we just instantiate the correct python object wrapper at runtime depending on the actual returned type. We use this lookup table to do so. */ static int TOTAL_CLASSES=0; /* This is a global reference to this module so classes can call each other. */ static PyObject *g_module = NULL; static struct python_wrapper_map_t { Object class_ref; PyTypeObject *python_type; } python_wrappers[%s]; /** This is a generic wrapper type */ typedef struct { PyObject_HEAD void *base; void *ctx; } Gen_wrapper; /* Create the relevant wrapper from the item based on the lookup table. */ Gen_wrapper *new_class_wrapper(Object item) { int i; Gen_wrapper *result; Object cls; /* Hack to get the current null_context */ void* null_context; int* tmp = talloc(NULL, int); null_context = talloc_parent(tmp); talloc_free(tmp); // Return None for a NULL pointer if(!item) { Py_INCREF(Py_None); return (Gen_wrapper *)Py_None; } // Search for subclasses for(cls=(Object)item->__class__; cls != cls->__super__; cls=cls->__super__) { for(i=0; ictx = talloc_asprintf(NULL, "new_class_wrapper %%s@%%p", NAMEOF(item), (void*)item); result->base = (void *)item; /* If its owned by the null_context it means that the function does not want to own the memory - we therefore steal it so it gets freed with the python object. */ if(talloc_parent(result->base) == null_context) talloc_steal(result->ctx, result->base); return result; } } } PyErr_Format(PyExc_RuntimeError, "Unable to find a wrapper for object %%s", NAMEOF(item)); return NULL; } static PyObject *resolve_exception(char **error_buff) { enum _error_type *type = aff4_get_current_error(error_buff); switch(*type) { case EProgrammingError: return PyExc_SystemError; case EKeyError: return PyExc_KeyError; case ERuntimeError: return PyExc_RuntimeError; case EWarning: return PyExc_AssertionError; default: return PyExc_RuntimeError; }; } static int type_check(PyObject *obj, PyTypeObject *type) { PyTypeObject *tmp; // Recurse through the inheritance tree and check if the types are expected if(obj) for(tmp = obj->ob_type; tmp && tmp != &PyBaseObject_Type; tmp = tmp->tp_base) { if(tmp == type) return 1; } return 0; } static int check_error() { if(!CheckError(EZero)) { char *buffer; PyObject *exception = resolve_exception(&buffer); PyErr_Format(exception, "%%s", buffer); ClearError(); return 1; } return 0; } #define CHECK_ERROR if(check_error()) goto error; """ % (len(self.classes)+1) def initialise_class(self, class_name, out, done = None): if done and class_name in done: return done.add(class_name) cls = self.classes[class_name] """ Write out class initialisation code into the main init function. """ if cls.is_active(): base_class = self.classes.get(cls.base_class_name) if base_class and base_class.is_active(): ## We have a base class - ensure it gets written out ## first: self.initialise_class(cls.base_class_name, out, done) ## Now assign ourselves as derived from them out.write(" %s_Type.tp_base = &%s_Type;" % ( cls.class_name, cls.base_class_name)) out.write(""" %(name)s_Type.tp_new = PyType_GenericNew; if (PyType_Ready(&%(name)s_Type) < 0) return; Py_INCREF((PyObject *)&%(name)s_Type); PyModule_AddObject(m, "%(name)s", (PyObject *)&%(name)s_Type); """ % {'name': cls.class_name}) def write(self, out): ## Prepare all classes for cls in self.classes.values(): cls.prepare() out.write(""" /********************************************************************** Autogenerated module %s This module was autogenerated from the following files: """ % self.name) for file in self.files: out.write("%s\n" % file) out.write("\nThis module implements the following classes:\n") out.write(self.__str__()) out.write("""***********************************************************************/ """) out.write(self.headers) out.write(self.private_functions()) for cls in self.classes.values(): if cls.is_active(): out.write("/******************** %s ***********************/" % cls.class_name) cls.struct(out) cls.prototypes(out) out.write("/*****************************************************\n Implementation\n******************************************************/\n\n") for cls in self.classes.values(): if cls.is_active(): cls.PyMethodDef(out) cls.code(out) cls.PyTypeObject(out) ## Write the module initializer out.write(""" static PyMethodDef %(module)s_methods[] = { {NULL} /* Sentinel */ }; PyMODINIT_FUNC init%(module)s(void) { /* Make sure threads are enabled */ PyEval_InitThreads(); /* create module */ PyObject *m = Py_InitModule3("%(module)s", %(module)s_methods, "%(module)s module."); PyObject *d = PyModule_GetDict(m); PyObject *tmp; g_module = m; """ % {'module': self.name}) ## The trick is to initialise the classes in order of their ## inheritance. The following code will order initializations ## according to their inheritance tree done = set() for class_name in self.classes.keys(): self.initialise_class(class_name, out, done) ## Add the constants in here for constant, type in self.constants: if type == 'integer': out.write(""" tmp = PyLong_FromUnsignedLongLong((int64_t)%s); \n""" % constant) elif type == 'string': out.write(" tmp = PyString_FromString((char *)%s); \n" % constant) else: out.write(" // I dont know how to convert %s type %s\n" % (constant, type)) continue out.write(""" PyDict_SetItemString(d, "%s", tmp); Py_DECREF(tmp);\n""" % (constant)) out.write(self.initialization()) out.write("}\n\n") class Type: interface = None buildstr = 'O' sense = 'IN' error_value = "return 0;" active = True def __init__(self, name, type): self.name = name self.type = type self.attributes = set() def comment(self): return "%s %s " % (self.type, self.name) def python_name(self): return self.name def python_proxy_post_call(self): """ This is called after a proxy call """ return '' def returned_python_definition(self, *arg, **kw): return self.definition(*arg, **kw) def definition(self, default=None, **kw): if default: return "%s %s=%s;\n" % (self.type, self.name, default) else: return "%s __attribute__((unused)) %s;\n" % (self.type, self.name) def byref(self): return "&%s" % self.name def call_arg(self): return self.name def pre_call(self, method): return '' def assign(self, call, method, target=None): return "Py_BEGIN_ALLOW_THREADS\n%s = %s;\nPy_END_ALLOW_THREADS\n" % (target or self.name, call) def post_call(self, method): if "DESTRUCTOR" in self.attributes: return "talloc_free(self->ctx); self->base = NULL;\n" return '' def from_python_object(self, source, destination, method, **kw): return '' def return_value(self, value): return "return %s;" % value def __str__(self): if self.name == 'func_return': return self.type if 'void' in self.type: return '' result = "%s : %s" % (self.type, self.name) return result class String(Type): interface = 'string' buildstr = 's' error_value = "return NULL;" def __init__(self, name, type): Type.__init__(self, name, type) self.length = "strlen(%s)" % name def byref(self): return "&%s" % self.name def to_python_object(self, name=None, result='py_result',**kw): name = name or self.name result = """PyErr_Clear(); if(!%(name)s) { PyErr_Format(PyExc_RuntimeError, "%(name)s is NULL"); goto error; } %(result)s = PyString_FromStringAndSize((char *)%(name)s, %(length)s);\nif(!%(result)s) goto error; """ % dict(name=name, result=result,length=self.length) if "BORROWED" not in self.attributes and 'BORROWED' not in kw: result += "talloc_unlink(NULL, %s);\n" % name return result def from_python_object(self, source, destination, method, context='NULL'): method.error_set = True return """ { char *buff; Py_ssize_t length; PyErr_Clear(); if(-1==PyString_AsStringAndSize(%(source)s, &buff, &length)) goto error; %(destination)s = talloc_size(%(context)s, length + 1); memcpy(%(destination)s, buff, length); %(destination)s[length]=0; } """ % dict(source = source, destination = destination, context =context) class ZString(String): interface = 'null_terminated_string' class BorrowedString(String): def to_python_object(self, name=None, result='py_result', **kw): name = name or self.name return "PyErr_Clear();\n" +\ "%s = PyString_FromStringAndSize((char *)%(name)s, %(length)s);\n" % dict( name=name, length=self.length, result=result) class Char_and_Length(Type): interface = 'char_and_length' buildstr = 's#' error_value = "return NULL;" def __init__(self, data, data_type, length, length_type): Type.__init__(self, data, data_type) self.name = data self.data_type=data_type self.length = length self.length_type = length_type def comment(self): return "%s %s, %s %s" % (self.data_type, self.name, self.length_type, self.length) def definition(self, default = '""', **kw): return "char *%s=%s; Py_ssize_t %s=strlen(%s);\n" % ( self.name, default, self.length, default) def byref(self): return "&%s, &%s" % (self.name, self.length) def call_arg(self): return "(%s)%s, (%s)%s" % (self.data_type, self.name, self.length_type, self.length) def to_python_object(self, name=None, result='py_result', **kw): return "PyErr_Clear();\n"\ "%s = PyString_FromStringAndSize(%s, %s);\nif(!%s) goto error;" % ( result, self.name, self.length, result); class Integer(Type): interface = 'integer' buildstr = 'K' def __init__(self, name,type): Type.__init__(self,name,type) self.type = 'uint64_t' self.original_type = type def definition(self, default = 0, **kw): return Type.definition(self, default) def to_python_object(self, name=None, result='py_result', **kw): name = name or self.name return "PyErr_Clear();\n%s = PyLong_FromLongLong(%s);\n" % (result, name) def from_python_object(self, source, destination, method, **kw): return "PyErr_Clear();\n"\ "%(destination)s = PyLong_AsUnsignedLong(%(source)s);\n" % dict( source = source, destination= destination) def comment(self): return "%s %s " % (self.original_type, self.name) class Integer32(Integer): buildstr = 'I' def __init__(self, name,type): Type.__init__(self,name,type) self.type = 'unsigned int ' self.original_type = type def to_python_object(self, name=None, result='py_result', **kw): name = name or self.name return "PyErr_Clear();\n%s = PyLong_FromLong(%s);\n" % (result, name) class Integer64(Integer): buildstr = 'K' type = 'unsigned int' class Char(Integer): buildstr = "s" interface = 'small_integer' def to_python_object(self, name = None, result = 'py_result', **kw): ## We really want to return a string here return """{ char *str_%(name)s = &%(name)s; PyErr_Clear(); %(result)s = PyString_FromStringAndSize(str_%(name)s, 1); if(!%(result)s) goto error; } """ % dict(result=result, name = name or self.name) def definition(self, default = '"\\x0"', **kw): ## Shut up unused warnings return "char %s __attribute__((unused))=0;\nchar *str_%s __attribute__((unused)) = %s;\n" % ( self.name,self.name, default) def byref(self): return "&str_%s" % self.name def pre_call(self, method): method.error_set = True return """ if(strlen(str_%(name)s)!=1) { PyErr_Format(PyExc_RuntimeError, "You must only provide a single character for arg %(name)r"); goto error; } %(name)s = str_%(name)s[0]; """ % dict(name = self.name) class StringOut(String): sense = 'OUT' class IntegerOut(Integer): sense = 'OUT_DONE' buildstr = '' def python_name(self): return None def byref(self): return '' def call_arg(self): return "&%s" % self.name class Integer32Out(Integer32): sense = 'OUT_DONE' buildstr = '' def python_name(self): return None def byref(self): return '' def call_arg(self): return "&%s" % self.name class Char_and_Length_OUT(Char_and_Length): sense = 'OUT_DONE' buildstr = 'l' def definition(self, default = 0, **kw): return "char *%s=NULL; Py_ssize_t %s=%s;\n" % ( self.name, self.length, default) + "PyObject *tmp_%s;\n" % self.name def python_name(self): return self.length def byref(self): return "&%s" % self.length def pre_call(self, method): return """PyErr_Clear(); tmp_%s = PyString_FromStringAndSize(NULL, %s); if(!tmp_%s) goto error; PyString_AsStringAndSize(tmp_%s, &%s, (Py_ssize_t *)&%s); """ % (self.name, self.length, self.name, self.name, self.name, self.length) def to_python_object(self, name=None, result='py_result', sense='in', **kw): name = name or self.name if 'results' in kw: kw['results'].pop(0) if sense=='proxied': return "py_%s = PyLong_FromLong(%s);\n" % (self.name, self.length) return """if(func_return > %(length)s) { func_return = 0; } _PyString_Resize(&tmp_%(name)s, func_return); \n%(result)s = tmp_%(name)s;\n""" % \ dict(name= name, result= result, length=self.length) def python_proxy_post_call(self, result='py_result'): return """ { char *tmp_buff; int tmp_len; if(-1==PyString_AsStringAndSize(%(result)s, &tmp_buff, &tmp_len)) goto error; memcpy(%(name)s,tmp_buff, tmp_len); Py_DECREF(%(result)s); %(result)s = PyLong_FromLong(tmp_len); } """ % dict(result = result, name=self.name) class TDB_DATA_P(Char_and_Length_OUT): bare_type = "TDB_DATA" def __init__(self, name, type): Type.__init__(self, name, type) def definition(self, default=None, **kw): return Type.definition(self) def byref(self): return "%s.dptr, &%s.dsize" % (self.name, self.name) def pre_call(self, method): return '' def call_arg(self): return Type.call_arg(self) def to_python_object(self, name=None,result='py_result', **kw): name = name or self.name return "PyErr_Clear();"\ "%s = PyString_FromStringAndSize((char *)%s->dptr, %s->dsize);"\ "\ntalloc_free(%s);" % (result, name, name, name) def from_python_object(self, source, destination, method, **kw): method.error_set = True return """ %(destination)s = talloc(self, %(bare_type)s); { Py_ssize_t tmp; char *buf; PyErr_Clear(); if(-1==PyString_AsStringAndSize(%(source)s, &buf, &tmp)) { goto error; } // Take a copy of the python string %(destination)s->dptr = talloc_memdup(%(destination)s, buf, tmp); %(destination)s->dsize = tmp; } // We no longer need the python object Py_DECREF(%(source)s); """ % dict(source = source, destination = destination, bare_type = self.bare_type) class TDB_DATA(TDB_DATA_P): def to_python_object(self, name = None, result='py_result', **kw): name = name or self.name return "PyErr_Clear();\n"\ "%s = PyString_FromStringAndSize((char *)%s.dptr, %s.dsize);\n" % ( result, name, name) class Void(Type): buildstr = '' error_value = "return;" original_type = '' def __init__(self, *args): Type.__init__(self, None, 'void *') def definition(self, default = None, **kw): return '' def to_python_object(self, name=None, result = 'py_result', **kw): return "Py_INCREF(Py_None); py_result = Py_None;\n" def call_arg(self): return "NULL" def byref(self): return None def assign(self, call, method, target=None): ## We dont assign the result to anything return "Py_BEGIN_ALLOW_THREADS\n%s;\nPy_END_ALLOW_THREADS\n" % call def return_value(self, value): return "return;" class StringArray(String): interface = 'array' buildstr = 'O' def definition(self, default = '""', **kw): return "char **%s=NULL; PyObject *py_%s=NULL;\n" % ( self.name, self.name) def byref(self): return "&py_%s" % (self.name) def from_python_object(self, source, destination, method, context='NULL'): method.error_set = True return """{ Py_ssize_t i,size=0; if(%(source)s) { if(!PySequence_Check(%(source)s)) { PyErr_Format(PyExc_ValueError, "%(destination)s must be a sequence"); goto error; } size = PySequence_Size(%(source)s); } %(destination)s = talloc_zero_array(NULL, char *, size + 1); for(i=0; ibase; """ % dict(source = source, destination = destination, type = self.type) def to_python_object(self, **kw): return '' def returned_python_definition(self, default = 'NULL', sense='in', **kw): return "%s %s;\n" % (self.type, self.name) def definition(self, default = 'NULL', sense='in', **kw): result = "Gen_wrapper *%s __attribute__((unused)) = %s;\n" % (self.name, default) if sense == 'in' and not 'OUT' in self.attributes: result += " %s __attribute__((unused)) call_%s;\n" % (self.type, self.name) return result def call_arg(self): return "call_%s" % self.name def pre_call(self, method): if 'OUT' in self.attributes or self.sense == 'OUT': return '' return """ if(!%(name)s || (PyObject *)%(name)s==Py_None) { call_%(name)s = NULL; } else if(!type_check((PyObject *)%(name)s,&%(type)s_Type)) { PyErr_Format(PyExc_RuntimeError, "%(name)s must be derived from type %(type)s"); goto error; } else { call_%(name)s = %(name)s->base; }\n""" % self.__dict__ def assign(self, call, method, target=None): method.error_set = True; args = dict(name = target or self.name, call = call, type = self.type) result = """{ Object returned_object; ClearError(); Py_BEGIN_ALLOW_THREADS returned_object = (Object)%(call)s; Py_END_ALLOW_THREADS CHECK_ERROR; """ % args ## Is NULL an acceptable return type? In some python code NULL ## can be returned (e.g. in iterators) but usually it should ## be converted to None. if "NULL_OK" in self.attributes: result += """if(returned_object == NULL) { %(name)s = NULL; """ % args else: result += """ // A NULL return without errors means we return None if(!returned_object) { %(name)s = (Gen_wrapper *)Py_None; Py_INCREF(Py_None); """ % args result += """ } else { //printf("%%s: Wrapping %%s@%%p\\n", __FUNCTION__, NAMEOF(returned_object), returned_object); %(name)s = new_class_wrapper(returned_object); if(!%(name)s) goto error; """ % args if "BORROWED" in self.attributes: result += " talloc_reference(%(name)s->ctx, %(name)s->base);\n" % args result += """ } } """ return result def to_python_object(self, name=None, result = 'py_result', sense='in', **kw): name = name or self.name args = dict(result=result, name = name) if sense=='proxied': return "%(result)s = (PyObject *)new_class_wrapper((Object)%(name)s);\n" % args return "%(result)s = (PyObject *)%(name)s;\n" % args class PointerWrapper(Wrapper): """ A pointer to a wrapped class """ def __init__(self, name, type): type = type.split()[0] Wrapper.__init__(self,name, type) def definition(self, default = 'NULL', sense='in', **kw): result = "Gen_wrapper *%s = %s;" % (self.name, default) if sense == 'in' and not 'OUT' in self.attributes: result += " %s *call_%s;\n" % (self.type, self.name) return result def pre_call(self, method): if 'OUT' in self.attributes or self.sense == 'OUT': return '' return """ if(!%(name)s || (PyObject *)%(name)s==Py_None) { call_%(name)s = NULL; } else if(!type_check((PyObject *)%(name)s,&%(type)s_Type)) { PyErr_Format(PyExc_RuntimeError, "%(name)s must be derived from type %(type)s"); goto error; } else { call_%(name)s = (%(type)s *)&%(name)s->base; }\n""" % self.__dict__ class StructWrapper(Wrapper): """ A wrapper for struct classes """ def assign(self, call, method, target = None): args = dict(name = target or self.name, call = call, type = self.type) result = """ PyErr_Clear(); %(name)s = (Gen_wrapper *)PyObject_New(py%(type)s, &%(type)s_Type); %(name)s->ctx = talloc_size(NULL, 1); %(name)s->base = %(call)s; """ % args if "NULL_OK" in self.attributes: result += "if(!%(name)s->base) { Py_DECREF(%(name)s); return NULL; }" % args result += """ // A NULL object gets translated to a None if(!%(name)s->base) { Py_DECREF(%(name)s); Py_INCREF(Py_None); %(name)s = (Gen_wrapper *)Py_None; } else { """ % args if "FOREIGN" in self.attributes: result += '// Not taking references to foreign memory\n' elif "BORROWED" in self.attributes: result += "talloc_reference(%(name)s->ctx, %(name)s->base);\n" % args else: result += "talloc_steal(%(name)s->ctx, %(name)s->base);\n" % args result += "}\n" return result def byref(self): return "&%s" % self.name def definition(self, default = 'NULL', sense='in', **kw): result = "Gen_wrapper *%s = %s;" % (self.name, default) if sense == 'in' and not 'OUT' in self.attributes: result += " %s *call_%s;\n" % (self.type, self.name) return result; class PointerStructWrapper(StructWrapper): def __init__(self, name, type): type = type.split()[0] Wrapper.__init__(self,name, type) class Timeval(Type): """ handle struct timeval values """ interface = 'numeric' buildstr = 'f' def definition(self, default = None, **kw): return "float %(name)s_flt; struct timeval %(name)s;\n" % self.__dict__ def byref(self): return "&%s_flt" % self.name def pre_call(self, method): return "%(name)s.tv_sec = (int)%(name)s_flt; %(name)s.tv_usec = (%(name)s_flt - %(name)s.tv_sec) * 1e6;\n" % self.__dict__ def to_python_object(self, name=None, result = 'py_result', **kw): name = name or self.name return """%(name)s_flt = (double)(%(name)s.tv_sec) + %(name)s.tv_usec; %(result)s = PyFloat_FromDouble(%(name)s_flt); """ % dict(name = name, result=result) class PyObject(Type): """ Accept an opaque python object """ interface = 'opaque' buildstr = 'O' def definition(self, default = 'NULL', **kw): self.default = default return 'PyObject *%(name)s = %(default)s;\n' % self.__dict__ def byref(self): return "&%s" % self.name type_dispatcher = { "IN char *": String, "IN unsigned char *": String, "unsigned char *": String, "char *": String, "ZString": ZString, "OUT char *": StringOut, "OUT unsigned char *": StringOut, "unsigned int": Integer, 'int': Integer, 'OUT uint64_t *': IntegerOut, 'OUT uint32_t *': Integer32Out, 'char': Char, 'void': Void, 'void *': Void, 'TDB_DATA *': TDB_DATA_P, 'TDB_DATA': TDB_DATA, 'uint64_t': Integer, 'uint32_t': Integer32, 'time_t': Integer32, 'uint16_t': Integer, 'uint8_t': Integer, 'int64_t': Integer, 'ssize_t': Integer, 'size_t': Integer, 'unsigned long int': Integer, 'struct timeval': Timeval, 'char **': StringArray, 'PyObject *': PyObject, } method_attributes = ['BORROWED', 'DESTRUCTOR','IGNORE'] def dispatch(name, type): if not type: return Void() m = re.match("struct ([a-zA-Z0-9]+)_t *", type) if m: type = m.group(1) type_components = type.split() attributes = set() if type_components[0] in method_attributes: attributes.add(type_components.pop(0)) type = " ".join(type_components) result = type_dispatcher[type](name, type) result.attributes = attributes return result class ResultException: value = 0 exception = "PyExc_IOError" def __init__(self, check, exception, message): self.check = check self.exception = exception self.message = message def write(self, out): out.write("\n//Handle exceptions\n") out.write("if(%s) {\n PyErr_Format(PyExc_%s, %s);\n goto error; \n}\n\n" % ( self.check, self.exception, self.message)) class Method: default_re = re.compile("DEFAULT\(([A-Z_a-z0-9]+)\) =(.+)") exception_re = re.compile("RAISES\(([^,]+),\s*([^\)]+)\) =(.+)") typedefed_re = re.compile(r"struct (.+)_t \*") def __init__(self, class_name, base_class_name, method_name, args, return_type, myclass = None): self.name = method_name ## myclass needs to be a class generator if not isinstance(myclass, ClassGenerator): raise RuntimeError("myclass must be a class generator") self.myclass = myclass self.docstring = '' self.defaults = {} self.exception = None self.error_set = False self.class_name = class_name self.base_class_name = base_class_name self.args = [] self.definition_class_name = class_name for type,name in args: self.add_arg(type, name) try: self.return_type = dispatch('func_return', return_type) self.return_type.attributes.add("OUT") self.return_type.original_type = return_type except KeyError: ## Is it a wrapped type? if return_type: log("Unable to handle return type %s.%s %s" % (self.class_name, self.name, return_type)) #pdb.set_trace() self.return_type = Void() def clone(self, new_class_name): self.find_optional_vars() result = self.__class__(new_class_name, self.base_class_name, self.name, [], 'void *', myclass = self.myclass) result.args = self.args result.return_type = self.return_type result.definition_class_name = self.definition_class_name result.defaults = self.defaults result.exception = self.exception return result def find_optional_vars(self): for line in self.docstring.splitlines(): m =self.default_re.search(line) if m: name = m.group(1) value = m.group(2) log("Setting default value for %s of %s" % (m.group(1), m.group(2))) self.defaults[name] = value m =self.exception_re.search(line) if m: self.exception = ResultException(m.group(1), m.group(2), m.group(3)) def write_local_vars(self, out): self.find_optional_vars() ## We do it in two passes - first mandatory then optional kwlist = """static char *kwlist[] = {""" ## Mandatory for type in self.args: python_name = type.python_name() if python_name and python_name not in self.defaults: kwlist += '"%s",' % python_name for type in self.args: python_name = type.python_name() if python_name and python_name in self.defaults: kwlist += '"%s",' % python_name kwlist += ' NULL};\n' for type in self.args: python_name = type.python_name() try: out.write(type.definition(default = self.defaults[python_name])) except KeyError: out.write(type.definition()) ## Make up the format string for the parse args in two pases parse_line = '' for type in self.args: python_name = type.python_name() if type.buildstr and python_name not in self.defaults: parse_line += type.buildstr parse_line += '|' for type in self.args: python_name = type.python_name() if type.buildstr and python_name in self.defaults: parse_line += type.buildstr if parse_line != '|': ## Now parse the args from python objects out.write(kwlist) out.write("\nif(!PyArg_ParseTupleAndKeywords(args, kwds, \"%s\", kwlist, " % parse_line) tmp = [] for type in self.args: ref = type.byref() if ref: tmp.append(ref) out.write(",".join(tmp)) self.error_set = True out.write("))\n goto error;\n\n") def error_condition(self): result = "" if "DESTRUCTOR" in self.return_type.attributes: result += "talloc_free(self->ctx); self->base = NULL;\n" return result +"return NULL;\n"; def write_definition(self, out): args = dict(method = self.name, class_name = self.class_name) out.write("\n/********************************************************\nAutogenerated wrapper for function:\n") out.write(self.comment()) out.write("********************************************************/\n") self._prototype(out) out.write("""{ PyObject *returned_result, *py_result; """ % args) out.write(self.return_type.definition()) self.write_local_vars( out); out.write("""// Make sure that we have something valid to wrap if(!self->base) return PyErr_Format(PyExc_RuntimeError, "%(class_name)s object no longer valid"); """ % args) ## Precall preparations out.write("// Precall preparations\n") out.write(self.return_type.pre_call(self)) for type in self.args: out.write(type.pre_call(self)) out.write("""// Check the function is implemented { void *method = ((%(def_class_name)s)self->base)->%(method)s; if(!method || (void *)unimplemented == (void *)method) { PyErr_Format(PyExc_RuntimeError, "%(class_name)s.%(method)s is not implemented"); goto error; } } """ % dict(def_class_name = self.definition_class_name, method=self.name, class_name = self.class_name)) out.write("\n// Make the call\n ClearError();") call = "((%s)self->base)->%s(((%s)self->base)" % (self.definition_class_name, self.name, self.definition_class_name) tmp = '' for type in self.args: tmp += ", " + type.call_arg() call += "%s)" % tmp ## Now call the wrapped function out.write(self.return_type.assign(call, self)) if self.exception: self.exception.write(out) self.error_set = True out.write("\n// Postcall preparations\n") ## Postcall preparations out.write(self.return_type.post_call(self)) for type in self.args: out.write(type.post_call(self)) ## Now assemble the results results = [self.return_type.to_python_object()] for type in self.args: if type.sense == 'OUT_DONE': results.append(type.to_python_object(results = results)) ## If all the results are returned by reference we dont need ## to prepend the void return value at all. if isinstance(self.return_type, Void) and len(results)>1: results.pop(0) out.write("\n// prepare results\n") ## Make a tuple of results and pass them back if len(results)>1: out.write("returned_result = PyList_New(0);\n") for result in results: out.write(result) out.write("PyList_Append(returned_result, py_result); Py_DECREF(py_result);\n"); out.write("return returned_result;\n") else: out.write(results[0]) ## This useless code removes compiler warnings out.write("returned_result = py_result;\nreturn returned_result;\n"); ## Write the error part of the function if self.error_set: out.write("\n// error conditions:\n") out.write("error:\n " + self.error_condition()); out.write("\n}\n\n") def add_arg(self, type, name): try: t = type_dispatcher[type](name, type) except KeyError: ## Sometimes types must be typedefed in advance try: m = self.typedefed_re.match(type) type = m.group(1) log( "Trying %s for %s" % (type, m.group(0))) t = type_dispatcher[type](name, type) except (KeyError, AttributeError): log( "Unable to handle type %s.%s %s" % (self.class_name, self.name, type)) return ## Here we collapse char * + int type interfaces into a ## coherent string like interface. try: previous = self.args[-1] if t.interface == 'integer' and \ previous.interface == 'string': ## We make a distinction between IN variables and OUT ## variables if previous.sense == 'OUT': cls = Char_and_Length_OUT else: cls = Char_and_Length cls = cls( previous.name, previous.type, name, type) self.args[-1] = cls return except IndexError: pass self.args.append(t) def comment(self): result = self.return_type.original_type+" "+self.class_name+"."+self.name+"(" args = [] for type in self.args: args.append( type.comment()) result += ",".join(args) + ");\n" return result def prototype(self, out): self._prototype(out) out.write(";\n") def _prototype(self, out): out.write("""static PyObject *py%(class_name)s_%(method)s(py%(class_name)s *self, PyObject *args, PyObject *kwds) """ % dict(method = self.name, class_name = self.class_name)) def __str__(self): result = "def %s %s(%s):" % ( self.return_type, self.name, ' , '.join([a.__str__() for a in self.args])) return result def PyMethodDef(self, out): docstring = self.comment() + "\n\n" + self.docstring out.write(' {"%s",(PyCFunction)py%s_%s, METH_VARARGS|METH_KEYWORDS, "%s"},\n' % ( self.name, self.class_name, self.name, escape_for_string(docstring))) class IteratorMethod(Method): """ A Method which implements an iterator """ def __init__(self, *args, **kw): Method.__init__(self, *args, **kw) ## Tell the return type that a NULL python return is ok self.return_type.attributes.add("NULL_OK") def _prototype(self, out): out.write("""static PyObject *py%(class_name)s_%(method)s(py%(class_name)s *self)""" % dict(method = self.name, class_name = self.class_name)) def __str__(self): result = "Iterator returning %s." % ( self.return_type) return result def PyMethodDef(self, out): ## This method should not go in the method table as its linked ## in directly pass class SelfIteratorMethod(IteratorMethod): def write_definition(self, out): args = dict(method = self.name, class_name = self.class_name) out.write("\n/********************************************************\nAutogenerated wrapper for function:\n") out.write(self.comment()) out.write("********************************************************/\n") self._prototype(out) out.write("""{ ((%(class_name)s)self->base)->%(method)s((%(class_name)s)self->base); return PyObject_SelfIter((PyObject *)self); } """ % args) class ConstructorMethod(Method): ## Python constructors are a bit different than regular methods def _prototype(self, out): out.write(""" static int py%(class_name)s_init(py%(class_name)s *self, PyObject *args, PyObject *kwds) """ % dict(method = self.name, class_name = self.class_name)) def write_destructor(self, out): ## We make sure that we unlink exactly the reference we need ## (the object will persist if it has some other ## references). Note that if we just used talloc_free here it ## will remove some random reference which may not actually be ## the reference we own (which is NULL). free = """ if(self->base) { //printf("Unlinking %s@%p\\n", NAMEOF(self->base), self->base); talloc_free(self->ctx); self->base=NULL; } """ out.write("""static void %(class_name)s_dealloc(py%(class_name)s *self) { %(free)s PyObject_Del(self); }\n """ % dict(class_name = self.class_name, free=free)) def error_condition(self): return "return -1;"; def write_definition(self, out): self._prototype(out) out.write("""{\n""") self.write_local_vars(out) ## Precall preparations for type in self.args: out.write(type.pre_call(self)) ## Now call the wrapped function out.write("\nself->ctx = talloc_strdup(NULL, \"%s\");" % self.class_name) out.write("\n ClearError();\nPy_BEGIN_ALLOW_THREADS\nself->base = CONSTRUCT(%s, %s, %s, self->ctx" % ( self.class_name, self.definition_class_name, self.name)) tmp = '' for type in self.args: tmp += ", " + type.call_arg() self.error_set = True out.write("""%s);\nPy_END_ALLOW_THREADS\n if(!CheckError(EZero)) { char *buffer; PyObject *exception = resolve_exception(&buffer); PyErr_Format(exception, "%%s", buffer); ClearError(); goto error; } else if(!self->base) { PyErr_Format(PyExc_IOError, "Unable to construct class %s"); goto error; } """ % (tmp, self.class_name)) out.write(" return 0;\n"); ## Write the error part of the function if self.error_set: out.write("error:\n " + self.error_condition()); out.write("\n}\n\n") class GetattrMethod(Method): def __init__(self, class_name, base_class_name, myclass): self.base_class_name = base_class_name self._attributes = [] self.error_set = True self.return_type = Void() self.myclass = myclass self.rename_class_name(class_name) def add_attribute(self, attr): if attr.name: self._attributes.append([self.class_name, attr]) def rename_class_name(self, new_name): """ This allows us to rename the class_name at a later stage. Required for late initialization of Structs whose name is not know until much later on. """ self.class_name = new_name self.name = "py%s_getattr" % new_name for x in self._attributes: x[0] = new_name def get_attributes(self): for class_name, attr in self._attributes: try: if not self.myclass.module.classes[attr.type].active: continue except KeyError: pass yield class_name, attr def __str__(self): result = "" for class_name, attr in self.get_attributes(): result += " %s\n" % attr.__str__() return result def clone(self, class_name): result = self.__class__(class_name, self.base_class_name, self.myclass) result._attributes = self._attributes[:] return result def prototype(self, out): if self.name: out.write(""" static PyObject *%(name)s(py%(class_name)s *self, PyObject *name); """ % self.__dict__) def built_ins(self, out): """ check for some built in attributes we need to support """ out.write(""" if(!strcmp(name, "__members__")) { PyObject *result = PyList_New(0); PyObject *tmp; PyMethodDef *i; if(!result) goto error; """) ## Add attributes for class_name, attr in self.get_attributes(): out.write(""" tmp = PyString_FromString("%(name)s"); PyList_Append(result, tmp); Py_DECREF(tmp); """ % dict(name = attr.name)) ## Add methods out.write(""" for(i=%s_methods; i->ml_name; i++) { tmp = PyString_FromString(i->ml_name); PyList_Append(result, tmp); Py_DECREF(tmp); }""" % self.class_name) out.write(""" return result; }\n""") def write_definition(self, out): if not self.name: return out.write(""" static PyObject *py%(class_name)s_getattr(py%(class_name)s *self, PyObject *pyname) { char *name; // Try to hand it off to the python native handler first PyObject *result = PyObject_GenericGetAttr((PyObject*)self, pyname); if(result) return result; PyErr_Clear(); // No - nothing interesting was found by python name = PyString_AsString(pyname); if(!self->base) return PyErr_Format(PyExc_RuntimeError, "Wrapped object (%(class_name)s.%(name)s) no longer valid"); if(!name) return NULL; """ % self.__dict__) self.built_ins(out) for class_name, attr in self.get_attributes(): ## what we want to assign if self.base_class_name: call = "(((%s)self->base)->%s)" % (class_name, attr.name) else: call = "(self->base->%s)" % (attr.name) out.write(""" if(!strcmp(name, "%(name)s")) { PyObject *py_result; %(python_def)s %(python_assign)s %(python_obj)s return py_result; }""" % dict(name = attr.name, python_obj = attr.to_python_object(), python_assign = attr.assign(call, self), python_def = attr.definition(sense='out'))) out.write(""" return PyObject_GenericGetAttr((PyObject *)self, pyname); """ % self.__dict__) ## Write the error part of the function if self.error_set: out.write("error:\n" + self.error_condition()); out.write("}\n\n") class ProxiedGetattr(GetattrMethod): def built_ins(self,out): out.write(""" if(!strcmp(name, "__members__")) { PyObject *result; PyObject *tmp; PyMethodDef *i; PyErr_Clear(); // Get the list of members from our proxied object result = PyObject_GetAttrString(self->base->proxied, name); if(!result) goto error; """) ## Add attributes for class_name, attr in self.get_attributes(): out.write(""" tmp = PyString_FromString("%(name)s"); PyList_Append(result, tmp); Py_DECREF(tmp); """ % dict(name = attr.name)) ## Add methods out.write(""" for(i=%s_methods; i->ml_name; i++) { tmp = PyString_FromString(i->ml_name); PyList_Append(result, tmp); Py_DECREF(tmp); } """ % self.class_name) out.write(""" return result; }\n""") out.write(""" /** Just try to get the attribute from our proxied object */ { PyObject *result = PyObject_GetAttrString(self->base->proxied, name); if(result) return result; } """) class ProxiedMethod(Method): def __init__(self, method, myclass): self.name = method.name self.method = method self.myclass = myclass self.class_name = method.class_name self.base_class_name = method.base_class_name self.args = method.args self.definition_class_name = method.definition_class_name self.return_type = method.return_type self.docstring = "Proxy for %s" % self.name self.defaults = {} self.exception = None self.error_set = False def get_name(self): return "py%(class_name)s_%(name)s" % dict(class_name =self.myclass.class_name, name = self.name) def _prototype(self, out): out.write(""" static %(return_type)s %(name)s(%(definition_class_name)s self""" % dict( return_type = self.return_type.original_type, class_name = self.myclass.class_name, method = self.name, name = self.get_name(), definition_class_name = self.definition_class_name)) for arg in self.args: out.write(", %s" % (arg.comment())) out.write(")") def prototype(self, out): self._prototype(out) out.write(";\n") def write_definition(self, out): self._prototype(out) self._write_definition(out) def _write_definition(self, out): ## We need to grab the GIL before we do anything out.write("""{ //Grab the GIL so we can do python stuff PyGILState_STATE gstate; gstate = PyGILState_Ensure(); """) out.write("{\nPyObject *py_result=NULL;\n") out.write('PyObject *method_name = PyString_FromString("%s");\n' % self.name) out.write(self.return_type.returned_python_definition()) for arg in self.args: out.write("PyObject *py_%s=NULL;\n" % arg.name) out.write("\n//Obtain python objects for all the args:\n") for arg in self.args: out.write(arg.to_python_object(result = "py_%s" % arg.name, sense='proxied', BORROWED=True)) out.write('if(!((%s)self)->proxied) {\n RaiseError(ERuntimeError, "No proxied object in %s"); goto error;\n}\n' % (self.myclass.class_name, self.myclass.class_name)) out.write("\n//Now call the method\n") out.write("""PyErr_Clear(); py_result = PyObject_CallMethodObjArgs(((%s)self)->proxied,method_name,""" % self.myclass.class_name) for arg in self.args: out.write("py_%s," % arg.name) ## Sentinal self.error_set = True out.write("""NULL); /** Check for python errors */ if(PyErr_Occurred()) { PyObject *exception_t, *exception, *tb; PyObject *str; char *str_c; char *error_str; enum _error_type *error_type = aff4_get_current_error(&error_str); // Fetch the exception state and convert it to a string: PyErr_Fetch(&exception_t, &exception, &tb); str = PyObject_Repr(exception); str_c = PyString_AsString(str); if(str_c) { strncpy(error_str, str_c, BUFF_SIZE-1); error_str[BUFF_SIZE-1]=0; *error_type = ERuntimeError; } Py_DECREF(str); goto error; } """); for arg in self.args: out.write(arg.python_proxy_post_call()) ## Now convert the python value back to a value out.write(self.return_type.from_python_object('py_result',self.return_type.name, self, context = "self")) out.write("if(py_result) { Py_DECREF(py_result);}\nPy_DECREF(method_name);\n\n"); out.write("PyGILState_Release(gstate);\n") ## Decref all our python objects: for arg in self.args: out.write("if(py_%s) { Py_DECREF(py_%s);}\n" %( arg.name, arg.name)) out.write(self.return_type.return_value('func_return')) if self.error_set: out.write("\nerror:\n") out.write("if(py_result) { Py_DECREF(py_result);}\nPy_DECREF(method_name);\n\n"); ## Decref all our python objects: for arg in self.args: out.write("if(py_%s) { Py_DECREF(py_%s);}\n" % (arg.name, arg.name)) out.write("PyGILState_Release(gstate);\n %s;\n" % self.error_condition()) out.write(" }\n}\n") def error_condition(self): return self.return_type.error_value class StructConstructor(ConstructorMethod): """ A constructor for struct wrappers - basically just allocate memory for the struct. """ def write_definition(self, out): out.write("""static int py%(class_name)s_init(py%(class_name)s *self, PyObject *args, PyObject *kwds) {\n""" % dict(method = self.name, class_name = self.class_name)) out.write("\nself->ctx = talloc_strdup(NULL, \"%s\");" % self.class_name) out.write("\nself->base = talloc(self->ctx, %s);\n" % self.class_name) out.write(" return 0;\n}\n\n") def write_destructor(self, out): out.write("""static void %(class_name)s_dealloc(py%(class_name)s *self) { talloc_free(self->ctx); }\n """ % dict(class_name = self.class_name)) class ProxyConstructor(ConstructorMethod): def write_destructor(self, out): out.write("""static void %(class_name)s_dealloc(py%(class_name)s *self) { if(self->base) { // Release the proxied object //Py_DECREF(self->base->proxied); talloc_free(self->ctx); self->base = NULL; } }\n static int %(class_name)s_destructor(void *this) { py%(class_name)s *self = (py%(class_name)s *)this; Py_DECREF(self->base->proxied); return 0; } """ % dict(class_name = self.class_name)) def initialise_attributes(self, out): attributes = self.myclass.module.classes[self.base_class_name].attributes.get_attributes() for definition_class_name, attribute in attributes: out.write(""" { // Converting from %(attribute_name)s PyErr_Clear(); PyObject *py_result = PyObject_GetAttrString(self->base->proxied, "%(name)s"); if(py_result) { %(type)s tmp; %(from_python_object)s; ((%(definition_class_name)s)self->base)->%(name)s = tmp; Py_DECREF(py_result); } PyErr_Clear(); }""" % dict(definition = attribute.definition(), name=attribute.name, attribute_name = attribute.__class__.__name__, type = attribute.type, definition_class_name = definition_class_name, from_python_object = attribute.from_python_object( 'py_result',"tmp", method=self, context = 'self->base'))) def write_constructor_proxy(self, out): ## Get the base_class constructor self.base_cons_method = ProxiedMethod(self.myclass.module.classes[self.base_class_name].constructor, self.myclass) self.base_cons_method._prototype(out) out.write("{\nPyObject *py_result;\n") out.write('PyObject *method_name;') out.write("%(class_name)s this = (%(class_name)s)self;\n" % self.__dict__) out.write("PyGILState_STATE gstate;\ngstate = PyGILState_Ensure();\n") out.write('method_name = PyString_FromString("__class__");\n') for arg in self.base_cons_method.args: out.write("PyObject *py_%s;\n" % arg.name) out.write("\n//Obtain python objects for all the args:\n") for arg in self.base_cons_method.args: out.write(arg.to_python_object(result = "py_%s" % arg.name, sense = 'proxied', BORROWED=True)) out.write('if(!((%s)self)->proxied) {\n RaiseError(ERuntimeError, "No proxied object in %s"); goto error;\n}\n' % (self.myclass.class_name, self.myclass.class_name)) out.write(""" // Enlarge the object size to accomodate the extended class self = talloc_realloc_size(self, self, sizeof(struct %(base_class_name)s_t)); """ % self.__dict__) out.write("\n//Now call the method\n") out.write("PyErr_Clear();\npy_result = PyObject_CallMethodObjArgs(((%s)self)->proxied,method_name," % self.myclass.class_name) call = '' for arg in self.base_cons_method.args: call += "py_%s," % arg.name ## Sentinal self.error_set = True call += """NULL""" out.write(call + ");\n"); out.write(""" if(!py_result && PyCallable_Check(this->proxied)) { PyErr_Clear(); py_result = PyObject_CallFunctionObjArgs(((%(name)s)self)->proxied, %(call)s); } /** Check for python errors */ if(PyErr_Occurred()) { PyObject *exception_t, *exception, *tb; PyObject *str; char *str_c; char *error_str; enum _error_type *error_type = aff4_get_current_error(&error_str); // Fetch the exception state and convert it to a string: PyErr_Fetch(&exception_t, &exception, &tb); str = PyObject_Repr(exception); str_c = PyString_AsString(str); if(str_c) { strncpy(error_str, str_c, BUFF_SIZE-1); error_str[BUFF_SIZE-1]=0; *error_type = ERuntimeError; } Py_DECREF(str); goto error; } // Take over the proxied object now this->proxied = py_result; """ % dict(name=self.myclass.class_name, call = call)); ## Now we try to populate the C struct slots with proxies of ## the python objects for class_name, attr in self.myclass.module.classes[\ self.base_class_name].attributes.get_attributes(): out.write(""" // Converting %(name)s from proxy: { PyObject *py_result = PyObject_GetAttrString(this->proxied, "%(name)s"); if(py_result) { """ % dict(name = attr.name)) out.write("{ %s " % attr.definition()) out.write(attr.from_python_object("py_result", "((%s)self)->%s" % (class_name, attr.name), self)) out.write("}\nPy_DECREF(py_result);\n}\n}\n") out.write("PyGILState_Release(gstate);\n") out.write("\n\nreturn self;\n") if self.error_set: out.write("error:\n PyGILState_Release(gstate);\ntalloc_free(self); return NULL;\n") out.write("}\n\n") def write_definition(self, out): self.write_constructor_proxy(out) out.write("""static int py%(class_name)s_init(py%(class_name)s *self, PyObject *args, PyObject *kwds) { PyGILState_STATE gstate = PyGILState_Ensure(); """ % dict(method = self.name, class_name = self.class_name)) self.write_local_vars(out) ## Precall preparations for type in self.args: out.write(type.pre_call(self)) ## Make up the call self.call = "talloc_memdup(NULL, &__%(class_name)s, sizeof(__%(class_name)s));\n" % self.__dict__ ## Now call the wrapped function out.write(""" self->base = %(call)s // Take over a copy of the proxied object self->base->proxied = proxied; /* We take a reference to the proxied object, and the proxied base class takes a reference. This way we (the python object) and the proxied C class can be freed independantly and only when both are freed the proxied object is freed. */ //Py_INCREF(proxied); Py_INCREF(proxied); talloc_set_destructor((void*)self->base, %(class_name)s_destructor); """ % self.__dict__) ## Install the handler for the constructor. FIXME - implement ## shortcut bypassing here so if the proxied class is itself a ## python binding to a C class and it does not override ## anything, we call directly into the original C class method ## instead of converting to python, and back. out.write("((%(definition_class_name)s)self->base)->%(name)s = %(func)s;\n" % dict( definition_class_name = self.base_cons_method.definition_class_name, name = self.base_cons_method.name, func = self.base_cons_method.get_name())) ## Now iterate over all our methods and install handlers: for method in self.myclass.methods: out.write(""" if(1 || PyDict_GetItemString(proxied->ob_type->tp_dict, "%(name)s")) { ((%(definition_class_name)s)self->base)->%(name)s = py%(class_name)s_%(name)s; } """ % dict(definition_class_name = method.definition_class_name, name = method.name, class_name = self.myclass.class_name)) ## Now fill in all attributes from the proxied object. Since ## the C struct attribute access is just memory access its ## difficult to trap it and refill attributes dynamically from ## the python object. Therefore for now we just read all ## attributes initially and populate the C struct with them. self.initialise_attributes(out) out.write("\n PyGILState_Release(gstate);\n return 0;\n"); ## Write the error part of the function if self.error_set: out.write("error:\n PyGILState_Release(gstate);\n " + self.error_condition()); out.write("\n}\n\n") class EmptyConstructor(ConstructorMethod): def write_definition(self, out): out.write("""static int py%(class_name)s_init(py%(class_name)s *self, PyObject *args, PyObject *kwds) {\n""" % dict(method = self.name, class_name = self.class_name)) out.write("""return 0;}\n\n""") class ClassGenerator: docstring = '' def __init__(self, class_name, base_class_name, module): self.class_name = class_name self.methods = [] self.module = module self.constructor = EmptyConstructor(class_name, base_class_name, "Con", [], '', myclass = self) self.base_class_name = base_class_name self.attributes = GetattrMethod(self.class_name, self.base_class_name, self) self.modifier = set() self.active = True self.iterator = None def prepare(self): """ This method is called just before we need to write the output and allows us to do any last minute fixups. """ pass def __str__(self): result = "#%s\n" % self.docstring result += "Class %s(%s):\n" % (self.class_name, self.base_class_name) result += " Constructor:%s\n" % self.constructor result += " Attributes:\n%s\n" % self.attributes result += " Methods:\n" for a in self.methods: result += " %s\n" % a.__str__() return result def is_active(self): """ Returns true if this class is active and should be generated """ if not self.active or self.modifier and \ ('PRIVATE' in self.modifier or 'ABSTRACT' in self.modifier): log("%s is not active %s" % (self.class_name, self.modifier)) return False return True def clone(self, new_class_name): """ Creates a clone of this class - usefull when implementing class extensions """ result = ClassGenerator(new_class_name, self.class_name, self.module) result.constructor = self.constructor.clone(new_class_name) result.methods = [ x.clone(new_class_name) for x in self.methods ] result.attributes = self.attributes.clone(new_class_name) return result def add_attribute(self, attr_name, attr_type, modifier): try: if not self.module.classes[attr_type].is_active(): return except KeyError: pass try: ## All attribute references are always borrowed - that ## means we dont want to free them after accessing them type_class = dispatch(attr_name, "BORROWED "+attr_type) except KeyError: log("Unknown attribute type %s for %s.%s" % (attr_type, self.class_name, attr_name)) return type_class.attributes.add(modifier) self.attributes.add_attribute(type_class) def add_constructor(self, method_name, args, return_type, docstring): if method_name.startswith("Con"): self.constructor = ConstructorMethod(self.class_name, self.base_class_name, method_name, args, return_type, myclass = self) self.constructor.docstring = docstring def struct(self,out): out.write("""\ntypedef struct { PyObject_HEAD %(class_name)s base; void *ctx; } py%(class_name)s;\n """ % dict(class_name=self.class_name)) def code(self, out): if not self.constructor: raise RuntimeError("No constructor found for class %s" % self.class_name) self.constructor.write_destructor(out) self.constructor.write_definition(out) if self.attributes: self.attributes.write_definition(out) for m in self.methods: m.write_definition(out) def initialise(self): return "python_wrappers[TOTAL_CLASSES].class_ref = (Object)&__%s;\n" \ "python_wrappers[TOTAL_CLASSES++].python_type = &%s_Type;\n" % ( self.class_name, self.class_name) def PyMethodDef(self, out): out.write("static PyMethodDef %s_methods[] = {\n" % self.class_name) for method in self.methods: method.PyMethodDef(out) out.write(" {NULL} /* Sentinel */\n};\n") def prototypes(self, out): """ Write prototype suitable for .h file """ out.write("""staticforward PyTypeObject %s_Type;\n""" % self.class_name) self.constructor.prototype(out) if self.attributes: self.attributes.prototype(out) for method in self.methods: method.prototype(out) def numeric_protocol_int(self): pass def numeric_protocol_nonzero(self): return """ static int %(class_name)s_nonzero(py%(class_name)s *v) { return v->base != 0; } """ % self.__dict__ def numeric_protocol(self, out): args = {'class':self.class_name} for type, func in [ ('nonzero', self.numeric_protocol_nonzero), ('int', self.numeric_protocol_int) ]: definition = func() if definition: out.write(definition) args[type] = "%s_%s" % (self.class_name,type) else: args[type] = '0' out.write(""" static PyNumberMethods %(class)s_as_number = { (binaryfunc) 0, /*nb_add*/ (binaryfunc) 0, /*nb_subtract*/ (binaryfunc) 0, /*nb_multiply*/ 0, /*nb_divide*/ 0, /*nb_remainder*/ 0, /*nb_divmod*/ 0, /*nb_power*/ (unaryfunc) 0, /*nb_negative*/ (unaryfunc) 0, /*tp_positive*/ (unaryfunc) 0, /*tp_absolute*/ (inquiry) %(nonzero)s, /*tp_nonzero*/ (unaryfunc) 0, /*nb_invert*/ 0, /*nb_lshift*/ (binaryfunc) 0, /*nb_rshift*/ 0, /*nb_and*/ 0, /*nb_xor*/ 0, /*nb_or*/ 0, /*nb_coerce*/ (unaryfunc) %(int)s, /*nb_int*/ 0, /*nb_long*/ 0, /*nb_float*/ 0, /*nb_oct*/ 0, /*nb_hex*/ 0, /* nb_inplace_add */ 0, /* nb_inplace_subtract */ 0, /* nb_inplace_multiply */ 0, /* nb_inplace_divide */ 0, /* nb_inplace_remainder */ 0, /* nb_inplace_power */ 0, /* nb_inplace_lshift */ 0, /* nb_inplace_rshift */ 0, /* nb_inplace_and */ 0, /* nb_inplace_xor */ 0, /* nb_inplace_or */ 0, /* nb_floor_divide */ 0, /* nb_true_divide */ 0, /* nb_inplace_floor_divide */ 0, /* nb_inplace_true_divide */ 0, /* nb_index */ }; """ % args) return "&%(class)s_as_number" % args def PyTypeObject(self, out): args = {'class':self.class_name, 'module': self.module.name, 'iterator': 0, 'iternext': 0, 'tp_str': 0, 'getattr_func': 0, 'docstring': "%s: %s" % (self.class_name, escape_for_string(self.docstring))} if self.attributes: args['getattr_func'] = self.attributes.name args['numeric_protocol'] = self.numeric_protocol(out) if "ITERATOR" in self.modifier: args['iterator'] = "PyObject_SelfIter" args['iternext'] = "py%s_iternext" % self.class_name if "SELF_ITER" in self.modifier: args['iterator'] = 'py%s___iter__' % self.class_name if "TP_STR" in self.modifier: args['tp_str'] = 'py%s___str__' % self.class_name out.write(""" static PyTypeObject %(class)s_Type = { PyObject_HEAD_INIT(NULL) 0, /* ob_size */ "%(module)s.%(class)s", /* tp_name */ sizeof(py%(class)s), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)%(class)s_dealloc,/* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_compare */ 0, /* tp_repr */ %(numeric_protocol)s, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ (reprfunc)%(tp_str)s, /* tp_str */ (getattrofunc)%(getattr_func)s, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ "%(docstring)s", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ (getiterfunc)%(iterator)s, /* tp_iter */ (iternextfunc)%(iternext)s,/* tp_iternext */ %(class)s_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)py%(class)s_init, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ }; """ % args ) class StructGenerator(ClassGenerator): """ A wrapper generator for structs """ def __init__(self, class_name, module): self.class_name = class_name self.methods = [] self.module = module self.base_class_name = None self.active = False self.modifier = set() self.constructor = None self.attributes = GetattrMethod(self.class_name, self.base_class_name, self) def prepare(self): ## This is needed for late stage initialization - sometimes ## our class_name is not know until now. if not self.constructor: self.constructor = StructConstructor(self.class_name, self.base_class_name, 'Con', [], "void", myclass = self) self.attributes.rename_class_name(self.class_name) for x in self.attributes._attributes: x[1].attributes.add('FOREIGN') def __str__(self): result = "#%s\n" % self.docstring result += "Struct %s:\n" % (self.class_name) result += "%s\n" % self.attributes return result def struct(self, out): out.write("""\ntypedef struct { PyObject_HEAD %(class_name)s *base; void *ctx; } py%(class_name)s;\n """ % dict(class_name=self.class_name)) def initialise(self): return '' class ProxyClassGenerator(ClassGenerator): def __init__(self, class_name, base_class_name, module, *args, **kwargs): ClassGenerator.__init__(self, class_name, base_class_name, module, *args, **kwargs) self.constructor = ProxyConstructor(self.class_name, self.base_class_name, '__init__', [('PyObject *', 'proxied')], 'void', myclass = self) self.module = module self.attributes = ProxiedGetattr(self.class_name, self.base_class_name, self) def initialise(self): return "INIT_CLASS(%(class_name)s);\n" % self.__dict__ + ClassGenerator.initialise(self) def struct(self, out): out.write(""" // The proxied type is an extension of the wrapped type with a pointer // to the proxied PyObject. CLASS(%(class_name)s, %(base_class_name)s) uint32_t magic; PyObject *proxied; END_CLASS VIRTUAL(%(class_name)s, %(base_class_name)s) { } END_VIRTUAL typedef struct { PyObject_HEAD %(class_name)s base; void *ctx; } py%(class_name)s;\n """ % self.__dict__) def PyMethodDef(self, out): out.write("static PyMethodDef %s_methods[] = {\n" % self.class_name) ## For now no methods out.write(" {NULL} /* Sentinel */\n};\n") class parser: class_re = re.compile(r"^([A-Z]+)?\s*CLASS\(([A-Z_a-z0-9]+)\s*,\s*([A-Z_a-z0-9]+)\)") method_re = re.compile(r"^\s*([0-9A-Z_a-z ]+\s+\*?)METHOD\(([A-Z_a-z0-9]+),\s*([A-Z_a-z0-9]+),?") enum_start_re = re.compile(r'enum ([0-9A-Z_a-z]+) {') enum_re = re.compile(r'([0-9A-Z_a-z]+) = [^,]+,') enum_end_re = re.compile('}') arg_re = re.compile(r"\s*([0-9A-Z a-z_]+\s+\*?)([0-9A-Za-z_]+),?") constant_re = re.compile(r"#define\s+([A-Z_0-9]+)\s+[^\s]+") struct_re = re.compile(r"([A-Z]+)?\s+(typedef )?struct\s+([A-Z_a-z0-9]+)\s+{") proxy_class_re = re.compile(r"^([A-Z]+)?\s*PROXY_CLASS\(([A-Za-z0-9]+)\)") end_class_re = re.compile("END_CLASS") attribute_re = re.compile(r"^\s*([0-9A-Z_a-z ]+\s+\*?)\s*([A-Z_a-z]+)\s*;") comment_re = re.compile(r"^\s*//") comment_start_re = re.compile(r"/\*+") comment_end_re = re.compile(r"\*+/") blank_line_re = re.compile("\s+") typedef_re = re.compile("typedef ([A-Za-z_0-9]+) +([A-Za-z_0-9]+) *;") bind_re = re.compile("BIND\(([a-zA-Z_0-9]+)\);") current_class = None def __init__(self, module, verbosity=0): self.module = module self.current_comment = '' self.verbosity = verbosity self.state = 'DEFAULT' ## this is a list of objects which are to be bound self.to_bind = [] global DEBUG DEBUG = verbosity io = StringIO.StringIO(""" // Base object CLASS(Object, Obj) END_CLASS """) self.parse_fd(io, '') def add_class(self, class_name, base_class_name, class_type, handler, docstring, modifier): try: self.current_class = self.module.classes[base_class_name].clone(class_name) except (KeyError, AttributeError): log("Base class %s is not defined !!!!" % base_class_name) self.current_class = class_type(class_name, base_class_name, self.module) ## Now add the new class to the module object self.current_class.docstring = docstring self.current_class.modifier = modifier self.module.add_class(self.current_class, handler) def parse_filenames(self, filenames): ## Be quiet for the first pass as many problems will be ## resolved on the second pass anyway. global DEBUG DEBUG = 0 for f in filenames: self._parse(f) DEBUG = self.verbosity log("Second pass: Consolidating definitions") for f in filenames: self._parse(f) def _parse(self, filename): fd = open(filename) self.parse_fd(fd, filename) fd.close() if filename not in self.module.files: self.module.headers += '#include "%s"\n' % filename self.module.files.append(filename) def parse_fd(self, fd, filename): self.current_class = None self.current_comment = '' while 1: line = fd.readline() if not line: break ## Handle binds m= self.bind_re.search(line) if m: print "Will bind %s" % m.group(1) self.to_bind.append(m.group(1)) continue ## Handle enums if self.state == 'DEFAULT': m = self.enum_start_re.search(line) if m: self.state = 'enum' type_dispatcher[m.group(1)] = type_dispatcher['int'] continue elif self.state == 'enum': m = self.enum_re.search(line) if m: self.module.add_constant(m.group(1), 'integer') if '}' in line: self.state = 'DEFAULT' continue ## Handle c++ style comments // m = self.comment_re.match(line) if m: self.current_comment = line[m.end():] while 1: line = fd.readline() m = self.comment_re.match(line) if not m: break self.current_comment += line[m.end():] ## Multiline C style comments m = self.comment_start_re.search(line) if m: line = line[m.end():] while 1: m = self.comment_end_re.search(line) if m: self.current_comment += line[:m.start()] line = fd.readline() break else: self.current_comment += line line = fd.readline() if not line: break ## Handle simple typdefs (as if typedef uint64_t my_fancy_type;) m = self.typedef_re.search(line) if m: ## We basically add a new type as a copy of the old ## type old, new = m.group(1), m.group(2) if old in type_dispatcher: type_dispatcher[new] = type_dispatcher[old] ## Handle constant #define m = self.constant_re.search(line) if m: ## We need to determine if it is a string or integer if re.search('"', line): ## Its a string self.module.add_constant(m.group(1), 'string') else: self.module.add_constant(m.group(1), 'integer') ## Wrap structs m = self.struct_re.search(line) if m: modifier = m.group(1) class_name = m.group(3) base_class_name = None ## Structs may be refered to as a pointer or absolute ## - its the same thing ultimatley. ## We only wrap structures which are explicitely bound if (modifier and 'BOUND' in modifier) or \ class_name in self.to_bind: self.add_class(class_name, base_class_name, StructGenerator, StructWrapper, self.current_comment, modifier) type_dispatcher["%s *" % class_name] = PointerStructWrapper continue m = self.class_re.search(line) if m: ## We need to make a new class now... We basically ## need to build on top of previously declared base ## classes - so we try to find base classes, clone ## them if possible: modifier = m.group(1) class_name = m.group(2) base_class_name = m.group(3) self.add_class(class_name, base_class_name, ClassGenerator, Wrapper, self.current_comment, modifier) type_dispatcher["%s *" % class_name] = PointerWrapper continue ## Make a proxy class for python callbacks m = self.proxy_class_re.search(line) if m: modifier = m.group(1) base_class_name = m.group(2) class_name = "Proxied%s" % base_class_name try: proxied_class = self.module.classes[base_class_name] except KeyError: raise RuntimeError("Need to create a proxy for %s but it has not been defined (yet). You must place the PROXIED_CLASS() instruction after the class definition" % base_class_name) self.current_class = ProxyClassGenerator(class_name, base_class_name, self.module) #self.current_class.constructor.args += proxied_class.constructor.args self.current_class.docstring = self.current_comment ## Create proxies for all these methods for method in proxied_class.methods: self.current_class.methods.append(ProxiedMethod(method, self.current_class)) self.module.add_class(self.current_class, Wrapper) ## Make sure that further lines are not interpreted as part of this class. self.current_class = None m = self.method_re.search(line) if self.current_class and m: args = [] method_name = m.group(3) return_type = m.group(1).strip() ## Ignore private methods if return_type.startswith("PRIVATE"): continue ## Now parse the args offset = m.end() while 1: m = self.arg_re.match(line[offset:]) if not m: ## Allow multiline definitions if there is \\ ## at the end of the line if line.strip().endswith("\\"): line = fd.readline() offset = 0 if line: continue break offset += m.end() args.append([m.group(1).strip(), m.group(2).strip()]) if return_type == self.current_class.class_name and \ method_name.startswith("Con"): self.current_class.add_constructor(method_name, args, return_type, self.current_comment) else: self.current_class.add_method(method_name, args, return_type, self.current_comment) m = self.attribute_re.search(line) if self.current_class and m: type = m.group(1) name = m.group(2) self.current_class.add_attribute(name, type) m = self.end_class_re.search(line) if m: ## Just clear the current class context self.current_class = None ## If this is a shadow file we do not include it from the ## main module. A shadow file is a way for us to redefine ## other C constructs which will be imported from an ## external header file so we can bind them slightly ## differently. if "this is a shadow file" in self.current_comment \ and filename not in self.module.files: self.module.files.append(filename) ## We only care about comment immediately above methods ## etc as we take them to be documentation. If we get here ## the comment is not above anything we care about - so we ## clear it: self.current_comment = '' def write(self, out): self.module.write(out) import lexer class EnumConstructor(ConstructorMethod): def write_destructor(self, out): out.write("""static void %(class_name)s_dealloc(py%(class_name)s *self) { Py_DECREF(self->value); PyObject_Del(self); }\n """ % dict(class_name = self.class_name)) def write_definition(self, out): self.myclass.modifier.add("TP_STR") self._prototype(out) out.write("""{ static char *kwlist[] = {"value", NULL}; if(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &self->value)) goto error; Py_INCREF(self->value); return 0; error: return -1; } static PyObject *py%(class_name)s___str__(py%(class_name)s *self) { PyObject *result = PyDict_GetItem(%(class_name)s_rev_lookup, self->value); if(result) { Py_INCREF(result); } else { result = PyObject_Str(self->value); } return result; } """ % self.__dict__) class Enum(StructGenerator): def __init__(self, name, module): StructGenerator.__init__(self, name, module) self.values = [] self.name = name self.attributes = None self.active = True def prepare(self): self.constructor = EnumConstructor(self.class_name, self.base_class_name, 'Con', [], "void", myclass = self) StructGenerator.prepare(self) def __str__(self): result = "Enum %s:\n" % (self.name) for attr in self.values: result += " %s\n" % attr.__str__() return result def struct(self,out): out.write("""\ntypedef struct { PyObject_HEAD PyObject *value; } py%(class_name)s;\n static PyObject *%(class_name)s_Dict_lookup; static PyObject *%(class_name)s_rev_lookup; """ % dict(class_name=self.class_name)) def PyMethodDef(self, out): out.write("static PyMethodDef %s_methods[] = {\n" % self.class_name) out.write(" {NULL} /* Sentinel */\n};\n") def numeric_protocol_nonzero(self): pass def numeric_protocol_int(self): return """ static PyObject *%(class_name)s_int(py%(class_name)s *self) { Py_INCREF(self->value); return self->value; } """ % self.__dict__ def initialise(self): result = """ %(class_name)s_Dict_lookup = PyDict_New(); %(class_name)s_rev_lookup = PyDict_New(); """ % self.__dict__ if self.values: result += "{ PyObject *tmp, *tmp2;\n" for attr in self.values: result += ''' tmp = PyLong_FromLong(%(value)s); tmp2 = PyString_FromString("%(value)s"); PyDict_SetItem(%(class_name)s_Dict_lookup, tmp2, tmp); PyDict_SetItem(%(class_name)s_rev_lookup, tmp, tmp2); Py_DECREF(tmp); Py_DECREF(tmp2); ''' % dict(value = attr, class_name=self.class_name) result += "};\n" return result class EnumType(Integer): buildstr = 'i' def __init__(self, name, type): Integer.__init__(self, name, type) self.type = type def to_python_object(self, name=None, result='py_result', **kw): name = name or self.name return """PyErr_Clear(); %s = PyObject_CallMethod(g_module, "%s", "K", (uint64_t)%s); """ % (result, self.type, name) def pre_call(self, method): method.error_set = True return """ // Check if the integer passed is actually a valid member of the enum // Enum value of 0 is always allowed if(%(name)s) { PyObject *py_%(name)s = PyLong_FromLong(%(name)s); PyObject *tmp = PyDict_GetItem(%(type)s_rev_lookup, py_%(name)s); Py_DECREF(py_%(name)s); if(!tmp) { PyErr_Format(PyExc_RuntimeError, "value %%lu is not valid for Enum %(type)s of arg '%(name)s'", (unsigned long)%(name)s); goto error; } Py_DECREF(tmp); } """ % self.__dict__ class HeaderParser(lexer.SelfFeederMixIn): tokens = [ [ 'INITIAL', r'#define\s+', 'PUSH_STATE', 'DEFINE' ], [ 'DEFINE', r'([A-Za-z_0-9]+)\s+[^\n]+', 'DEFINE,POP_STATE', None ], [ 'DEFINE', r'\n', 'POP_STATE', None], ## Recognize ansi c comments [ '.', r'/\*(.)', 'PUSH_STATE', 'COMMENT' ], [ 'COMMENT', r'(.+?)\*/\s+', 'COMMENT_END,POP_STATE', None], [ 'COMMENT', r'(.+)', 'COMMENT', None], ## And c++ comments [ '.', r'//([^\n]+)', 'COMMENT', None], ## An empty line clears the current comment [ '.', r'\r?\n\r?\n', 'CLEAR_COMMENT', None], ## Ignore whitespace [ '.', r'\s+', 'SPACE', None ], [ '.', r'\\\n', 'SPACE', None ], ## Recognize CLASS() definitions [ 'INITIAL', r"^([A-Z]+)?\s*CLASS\(([A-Z_a-z0-9]+)\s*,\s*([A-Z_a-z0-9]+)\)", 'PUSH_STATE,CLASS_START', 'CLASS'], [ 'CLASS', r"^\s*(FOREIGN|ABSTRACT|PRIVATE)?([0-9A-Z_a-z ]+( |\*))METHOD\(([A-Z_a-z0-9]+),\s*([A-Z_a-z0-9]+),?", "PUSH_STATE,METHOD_START", "METHOD"], [ 'METHOD', r"\s*([0-9A-Z a-z_]+\s+\*?\*?)([0-9A-Za-z_]+),?", "METHOD_ARG", None ], [ 'METHOD', r'\);', 'POP_STATE,METHOD_END', None], [ 'CLASS', r"^\s*(FOREIGN|ABSTRACT)?([0-9A-Z_a-z ]+\s+\*?)\s*([A-Z_a-z0-9]+)\s*;", 'CLASS_ATTRIBUTE', None], [ 'CLASS', "END_CLASS", 'END_CLASS,POP_STATE', None], ## Recognize struct definitions (With name) [ 'INITIAL', "([A-Z_a-z0-9 ]+)?struct\s+([A-Z_a-z0-9]+)\s+{", 'PUSH_STATE,STRUCT_START', 'STRUCT'], ## Without name (using typedef) [ 'INITIAL', "typedef\s+struct\s+{", 'PUSH_STATE,TYPEDEF_STRUCT_START', 'STRUCT'], [ 'STRUCT', r"^\s*([0-9A-Z_a-z ]+\s+\*?)\s*([A-Z_a-z0-9]+)\s*;", 'STRUCT_ATTRIBUTE', None], [ 'STRUCT', r"^\s*([0-9A-Z_a-z ]+)\*\s+([A-Z_a-z0-9]+)\s*;", 'STRUCT_ATTRIBUTE_PTR', None], ## Struct ended with typedef [ 'STRUCT', '}\s+([0-9A-Za-z_]+);', 'POP_STATE,TYPEDEF_STRUCT_END', None], [ 'STRUCT', '}', 'POP_STATE,STRUCT_END', None], ## Handle recursive struct or union definition (At the moment ## we cant handle them at all) [ '(RECURSIVE_)?STRUCT', '(struct|union)\s+([_A-Za-z0-9]+)?\s*{', 'PUSH_STATE', 'RECURSIVE_STRUCT'], [ 'RECURSIVE_STRUCT', '}\s+[0-9A-Za-z]+', 'POP_STATE', None], ## Process enums (2 forms - named and typedefed) [ 'INITIAL', r'enum\s+([0-9A-Za-z_]+)\s+{', 'PUSH_STATE,ENUM_START', 'ENUM' ], ## Unnamed [ 'INITIAL', r'typedef\s+enum\s+{', 'PUSH_STATE,TYPEDEF_ENUM_START', 'ENUM' ], [ 'ENUM', r'([0-9A-Za-z_]+)\s+=[^\n]+', 'ENUM_VALUE', None], ## Typedefed ending [ 'ENUM', r'}\s+([0-9A-Za-z_]+);', 'POP_STATE,TYPEDEFED_ENUM_END', None], [ 'ENUM', r'}', 'POP_STATE,ENUM_END', None], [ 'INITIAL', r'BIND_STRUCT\(([0-9A-Za-z_ \*]+)\)', 'BIND_STRUCT', None], ## A simple typedef of one type for another type: [ 'INITIAL', r"typedef ([A-Za-z_0-9]+) +([^;]+);", 'SIMPLE_TYPEDEF', None], ## Handle proxied directives [ 'INITIAL', r"PROXY_CLASS\(([A-Za-z0-9]+)\);", 'PROXY_CLASS', None], ] def __init__(self, name, verbose = 1): self.module = Module(name) lexer.SelfFeederMixIn.__init__(self, verbose = 0) io = StringIO.StringIO(""" // Base object CLASS(Object, Obj) END_CLASS """) self.parse_fd(io) current_comment = '' def COMMENT(self, t, m): self.current_comment += m.group(1) + "\n" def COMMENT_END(self, t, m): self.current_comment += m.group(1) def CLEAR_COMMENT(self, t, m): self.current_comment = '' def DEFINE(self, t, m): line = m.group(0) line = line.split('/*')[0] if '"' in line: type = 'string' else: type = 'integer' name = m.group(1).strip() if len(name)>3 and name[0]!='_' and name==name.upper(): self.module.add_constant(name, type) current_class = None def CLASS_START(self, t, m): class_name = m.group(2).strip() base_class_name = m.group(3).strip() try: self.current_class = self.module.classes[base_class_name].clone(class_name) except (KeyError, AttributeError): log("Base class %s is not defined !!!!" % base_class_name) self.current_class = ClassGenerator(class_name, base_class_name, self.module) self.current_class.docstring = self.current_comment self.current_class.modifier.add(m.group(1)) self.module.add_class(self.current_class, Wrapper) type_dispatcher["%s *" % class_name] = PointerWrapper current_method = None def METHOD_START(self, t, m): return_type = m.group(2).strip() method_name = m.group(5).strip() modifier = m.group(1) or '' if 'PRIVATE' in modifier: return ## Is it a regular method or a constructor? self.current_method = Method if return_type == self.current_class.class_name and \ method_name.startswith("Con"): self.current_method = ConstructorMethod elif method_name == 'iternext': self.current_method = IteratorMethod self.current_class.modifier.add("ITERATOR") elif method_name == '__iter__': self.current_method = SelfIteratorMethod self.current_class.modifier.add("SELF_ITER") elif method_name == '__str__': self.current_class.modifier.add("TP_STR") self.current_method = self.current_method(self.current_class.class_name, self.current_class.base_class_name, method_name, [], return_type, myclass = self.current_class) self.current_method.docstring = self.current_comment self.current_method.modifier = modifier def METHOD_ARG(self, t, m): name = m.group(2).strip() type = m.group(1).strip() if self.current_method: self.current_method.add_arg(type, name) def METHOD_END(self, t, m): if not self.current_method: return if isinstance(self.current_method, ConstructorMethod): self.current_class.constructor = self.current_method else: found = False for i in range(len(self.current_class.methods)): ## Try to replace existing methods with this new method method = self.current_class.methods[i] if method.name == self.current_method.name: self.current_class.methods[i] = self.current_method self.current_method = None return ## Method does not exist, just add to the end self.current_class.methods.append(self.current_method) self.current_method = None def CLASS_ATTRIBUTE(self, t, m): modifier = m.group(1) or '' type = m.group(2).strip() name = m.group(3).strip() self.current_class.add_attribute(name, type, modifier) def END_CLASS(self, t, m): self.current_class = None current_struct = None def STRUCT_START(self, t, m): self.current_struct = StructGenerator(m.group(2).strip(), self.module) self.current_struct.docstring = self.current_comment self.current_struct.modifier.add(m.group(1)) def TYPEDEF_STRUCT_START(self, t, m): self.current_struct = StructGenerator(None, self.module) self.current_struct.docstring = self.current_comment def STRUCT_ATTRIBUTE(self, t, m): name = m.group(2).strip() type = m.group(1).strip() self.current_struct.add_attribute(name, type, '') def STRUCT_ATTRIBUTE_PTR(self, t, m): type = "%s *" % m.group(1).strip() name = m.group(2).strip() self.current_struct.add_attribute(name, type, '') def STRUCT_END(self, t, m): self.module.add_class(self.current_struct, StructWrapper) type_dispatcher["%s *" % self.current_struct.class_name] = PointerStructWrapper self.current_struct = None def TYPEDEF_STRUCT_END(self, t, m): self.current_struct.class_name = m.group(1).strip() self.STRUCT_END(t, m) current_enum = None def ENUM_START(self, t, m): self.current_enum = Enum(m.group(1).strip(), self.module) def TYPEDEF_ENUM_START(self, t, m): self.current_enum = Enum(None, self.module) def ENUM_VALUE(self, t, m): self.current_enum.values.append(m.group(1).strip()) def ENUM_END(self, t, m): self.module.classes[self.current_enum.name] = self.current_enum ## For now we just treat enums as an integer, and also add ## them to the constant table. In future it would be nice to ## have them as a proper python object so we can override ## __str__ and __int__. for attr in self.current_enum.values: self.module.add_constant(attr, 'integer') #type_dispatcher[self.current_enum.name] = Integer type_dispatcher[self.current_enum.name] = EnumType self.current_enum = None def TYPEDEFED_ENUM_END(self, t, m): self.current_enum.name = self.current_enum.class_name = m.group(1) self.ENUM_END(t, m) def BIND_STRUCT(self, t, m): self.module.classes[m.group(1)].active = True def SIMPLE_TYPEDEF(self, t, m): ## We basically add a new type as a copy of the old ## type old, new = m.group(1).strip(), m.group(2).strip() if old in type_dispatcher: type_dispatcher[new] = type_dispatcher[old] def PROXY_CLASS(self, t, m): base_class_name = m.group(1).strip() class_name = "Proxied%s" % base_class_name try: proxied_class = self.module.classes[base_class_name] except KeyError: raise RuntimeError("Need to create a proxy for %s but it has not been defined (yet). You must place the PROXIED_CLASS() instruction after the class definition" % base_class_name) current_class = ProxyClassGenerator(class_name, base_class_name, self.module) #self.current_class.constructor.args += proxied_class.constructor.args current_class.docstring = self.current_comment ## Create proxies for all these methods for method in proxied_class.methods: current_class.methods.append(ProxiedMethod(method, current_class)) self.module.add_class(current_class, Wrapper) def parse_filenames(self, filenames): for f in filenames: self._parse(f) ## Second pass for f in filenames: self._parse(f) def _parse(self, filename): fd = open(filename) self.parse_fd(fd) fd.close() if filename not in self.module.files: self.module.headers += '#include "%s"\n' % filename self.module.files.append(filename) def write(self, out): self.module.write(out) if __name__ == '__main__': p = HeaderParser('pyaff4', verbose = 1) for arg in sys.argv[1:]: p.parse_fd(open(arg)) log("second parse") for arg in sys.argv[1:]: p.parse_fd(open(arg)) pdb.set_trace() p.write(sys.stdout) # p = parser(Module("pyaff4")) # for arg in sys.argv[1:]: # p.parse(arg) # log("second parse") # p.parse(arg) # p.write(sys.stdout)