diff --git a/bootstrap.py b/bootstrap.py index 1e7d0e71..c711fc05 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -1,40 +1,4 @@ import sys -import imp import uwsgi -class uWSGISymbolImporter(object): - - def find_module(self, fullname, path=None): - self.path = None - symname = fullname.replace('.','_') - try: - self.symbol = "%s_py" % symname - self.code = uwsgi.embedded_data(self.symbol) - return self - except: - try: - self.symbol = "%s___init___py" % symname - self.code = uwsgi.embedded_data(self.symbol) - self.path = "sym://%s" % symname - return self - except: - pass - - def load_module(self, fullname): - if fullname in sys.modules: - mod = sys.modules[fullname] - else: - mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) - - mod.__file__ = "sym://%s" % self.symbol - mod.__path__ = self.path - mod.__name__ = fullname - mod.__loader__ = self - mod.__package__ = '.'.join(fullname.split('.')[:-1]) - - exec uwsgi.embedded_data(self.symbol) in mod.__dict__ - - return mod - - -sys.meta_path.append(uWSGISymbolImporter()) +sys.meta_path.insert(0, uwsgi.SymbolsImporter()) diff --git a/django/uwsgitemplate.py b/django/uwsgitemplate.py index e0d5058b..525c155d 100644 --- a/django/uwsgitemplate.py +++ b/django/uwsgitemplate.py @@ -16,7 +16,7 @@ class Loader(BaseLoader): is_usable = on_uwsgi def symbolize(self, name): - return name.replace('.','_').replace('/','_') + return name.replace('.','_').replace('/','_').replace('-','_') def load_template_source(self, template_name, template_dirs=None): filename = 'templates_' + self.symbolize(template_name) diff --git a/plugins/python/python_plugin.c b/plugins/python/python_plugin.c index 3561658b..27973b7d 100644 --- a/plugins/python/python_plugin.c +++ b/plugins/python/python_plugin.c @@ -21,6 +21,7 @@ struct option uwsgi_python_options[] = { {"pythonpath", required_argument, 0, LONG_ARGS_PYTHONPATH}, {"python-path", required_argument, 0, LONG_ARGS_PYTHONPATH}, {"pymodule-alias", required_argument, 0, LONG_ARGS_PYMODULE_ALIAS}, + {"post-pymodule-alias", required_argument, 0, LONG_ARGS_POST_PYMODULE_ALIAS}, {"import", required_argument, 0, LONG_ARGS_PYIMPORT}, {"pyimport", required_argument, 0, LONG_ARGS_PYIMPORT}, {"py-import", required_argument, 0, LONG_ARGS_PYIMPORT}, @@ -699,6 +700,9 @@ int uwsgi_python_manage_options(int i, char *optarg) { case LONG_ARGS_PYIMPORT: uwsgi_string_new_list(&up.import_list, optarg); return 1; + case LONG_ARGS_POST_PYMODULE_ALIAS: + uwsgi_string_new_list(&up.post_pymodule_alias, optarg); + return 1; case LONG_ARGS_PYTHONPATH: if (glob(optarg, GLOB_MARK, NULL, &g)) { uwsgi_string_new_list(&up.python_path, optarg); @@ -790,16 +794,22 @@ char *uwsgi_pythonize(char *orig) { void uwsgi_python_init_apps() { + if (uwsgi.async > 1) { up.current_recursion_depth = uwsgi_malloc(sizeof(int)*uwsgi.async); up.current_frame = uwsgi_malloc(sizeof(struct _frame)*uwsgi.async); } init_pyargv(); + #ifdef UWSGI_MINTERPRETERS init_uwsgi_embedded_module(); #endif +#ifdef __linux__ + uwsgi_init_symbol_import(); +#endif + if (up.test_module != NULL) { if (PyImport_ImportModule(up.test_module)) { exit(0); @@ -821,6 +831,7 @@ void uwsgi_python_init_apps() { up.loaders[LOADER_CALLABLE] = uwsgi_callable_loader; up.loaders[LOADER_STRING_CALLABLE] = uwsgi_string_callable_loader; + struct uwsgi_string_list *upli = up.import_list; while(upli) { if (strchr(upli->value, '/') || uwsgi_endswith(upli->value, ".py")) { @@ -834,6 +845,43 @@ void uwsgi_python_init_apps() { upli = upli->next; } + struct uwsgi_string_list *uppa = up.post_pymodule_alias; + PyObject *modules = PyImport_GetModuleDict(); + PyObject *tmp_module; + while(uppa) { + // split key=value + char *value = strchr(uppa->value, '='); + if (!value) { + uwsgi_log("invalid pymodule-alias syntax\n"); + continue; + } + value[0] = 0; + if (!strchr(value + 1, '/')) { + // this is a standard pymodule + tmp_module = PyImport_ImportModule(value + 1); + if (!tmp_module) { + PyErr_Print(); + exit(1); + } + + PyDict_SetItemString(modules, uppa->value, tmp_module); + } + else { + // this is a filepath that need to be mapped + tmp_module = uwsgi_pyimport_by_filename(uppa->value, value + 1); + if (!tmp_module) { + PyErr_Print(); + exit(1); + } + } + uwsgi_log("mapped virtual pymodule \"%s\" to real pymodule \"%s\"\n", uppa->value, value + 1); + // reset original value + value[0] = '='; + + uppa = uppa->next; + } + + if (up.wsgi_config != NULL) { init_uwsgi_app(LOADER_UWSGI, up.wsgi_config, uwsgi.wsgi_req, up.main_thread); } diff --git a/plugins/python/symimporter.c b/plugins/python/symimporter.c new file mode 100644 index 00000000..fd487fa8 --- /dev/null +++ b/plugins/python/symimporter.c @@ -0,0 +1,219 @@ +#include "uwsgi_python.h" + +extern struct uwsgi_server uwsgi; +extern struct uwsgi_python up; + + +struct _symimporter { + PyObject_HEAD; +} uwsgi_symbol_importer_object; + +static char *symbolize(char *name) { + + char *base = uwsgi_concat2(name, ""); + char *ptr = base; + while(*ptr != 0) { + if (*ptr == '.') { + *ptr = '_'; + } + ptr++; + } + + return base; +} + +static char *name_to_symbol_module(char *name, char *what) { + + char *symbol = uwsgi_concat4("_binary_", name, "_py_", what); + char *sym_ptr_start = dlsym(RTLD_DEFAULT, symbol); + free(symbol); + return sym_ptr_start; +} + +static char *name_to_symbol_pkg(char *name, char *what) { + + char *symbol = uwsgi_concat4("_binary_", name, "___init___py_", what); + char *sym_ptr_start = dlsym(RTLD_DEFAULT, symbol); + free(symbol); + return sym_ptr_start; +} + +static PyObject* symimporter_find_module(PyObject *self, PyObject *args) { + + char *fullname; + PyObject *path = NULL; + + if (!PyArg_ParseTuple(args, "s|O:find_module", &fullname, &path)) { + return NULL; + } + + char *fullname2 = symbolize(fullname); + + char *code_start = name_to_symbol_module(fullname2, "start"); + if (code_start) { + free(fullname2); + Py_INCREF(self); + return self; + } + code_start = name_to_symbol_pkg(fullname2, "start"); + if (code_start) { + free(fullname2); + Py_INCREF(self); + return self; + } + + + free(fullname2); + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject* symimporter_load_module(PyObject *self, PyObject *args) { + + char *code_start; + char *code_end; + char *fullname; + char *source; + char *modname; + PyObject *code; + if (!PyArg_ParseTuple(args, "s:load_module", &fullname)) { + return NULL; + } + + char *fullname2 = symbolize(fullname); + + code_start = name_to_symbol_module(fullname2, "start"); + if (code_start) { + code_end = name_to_symbol_module(fullname2, "end"); + if (code_end) { + PyObject *mod = PyImport_AddModule(fullname); + if (!mod) goto clear; + PyObject *dict = PyModule_GetDict(mod); + if (!dict) goto clear; + + PyDict_SetItemString(dict, "__loader__", self); + + source = uwsgi_concat2n(code_start, code_end-code_start, "", 0); + modname = uwsgi_concat3("sym://", fullname2, "_py"); + + code = Py_CompileString(source, modname, Py_file_input); + mod = PyImport_ExecCodeModuleEx(fullname, code, modname); + + Py_DECREF(code); + free(source); + free(modname); + free(fullname2); + return mod; + } + } + + code_start = name_to_symbol_pkg(fullname2, "start"); + if (code_start) { + code_end = name_to_symbol_pkg(fullname2, "end"); + if (code_end) { + PyObject *mod = PyImport_AddModule(fullname); + if (!mod) goto clear; + PyObject *dict = PyModule_GetDict(mod); + if (!dict) goto clear; + + source = uwsgi_concat2n(code_start, code_end-code_start, "", 0); + modname = uwsgi_concat3("sym://", symbolize(fullname), "___init___py"); + + PyObject *pkgpath = Py_BuildValue("[O]", PyString_FromString(modname)); + + PyDict_SetItemString(dict, "__path__", pkgpath); + PyDict_SetItemString(dict, "__loader__", self); + + code = Py_CompileString(source, modname, Py_file_input); + mod = PyImport_ExecCodeModuleEx(fullname, code, modname); + + Py_DECREF(code); + free(source); + free(modname); + free(fullname2); + return mod; + } + } + +clear: + free(fullname2); + Py_INCREF(Py_None); + return Py_None; +} + +static PyMethodDef symimporter_methods[] = { + {"find_module", symimporter_find_module, METH_VARARGS}, + {"load_module", symimporter_load_module, METH_VARARGS}, +}; + +static void uwsgi_symimporter_free(struct _symimporter *self) { + PyObject_Del(self); +} + + +static PyTypeObject SymImporter_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "uwsgi.SymbolsImporter", + sizeof(struct _symimporter), + 0, /* tp_itemsize */ + (destructor) uwsgi_symimporter_free, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, + "uwsgi symbols importer", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + symimporter_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; + +int uwsgi_init_symbol_import() { + + + if (PyType_Ready(&SymImporter_Type) < 0) { + uwsgi_log("unable to initialize symbols importer module\n"); + exit(1); + } + + PyObject *uwsgi_em = PyImport_ImportModule("uwsgi"); + if (!uwsgi_em) { + uwsgi_log("unable to get uwsgi module\n"); + exit(1); + } + + if (PyModule_AddObject(uwsgi_em, "SymbolsImporter", + (PyObject *)&SymImporter_Type) < 0) { + uwsgi_log("unable to initialize symbols importer module\n"); + exit(1); + } + + return 0; + +} + diff --git a/plugins/python/uwsgi_python.h b/plugins/python/uwsgi_python.h index d59a28cf..4fb862ae 100644 --- a/plugins/python/uwsgi_python.h +++ b/plugins/python/uwsgi_python.h @@ -14,6 +14,7 @@ #define LONG_ARGS_PYMODULE_ALIAS LONG_ARGS_PYTHON_BASE + 4 #define LONG_ARGS_RELOAD_OS_ENV LONG_ARGS_PYTHON_BASE + 5 #define LONG_ARGS_PYIMPORT LONG_ARGS_PYTHON_BASE + 6 +#define LONG_ARGS_POST_PYMODULE_ALIAS LONG_ARGS_PYTHON_BASE + 7 #if PY_MINOR_VERSION == 4 && PY_MAJOR_VERSION == 2 #define Py_ssize_t ssize_t @@ -105,6 +106,7 @@ struct uwsgi_python { struct uwsgi_string_list *python_path; struct uwsgi_string_list *import_list; + struct uwsgi_string_list *post_pymodule_alias; PyObject *loader_dict; PyObject* (*loaders[LOADER_MAX]) (void *); @@ -229,3 +231,7 @@ void simple_reset_ts(struct wsgi_request *, struct uwsgi_app *); int uwsgi_python_profiler_call(PyObject *, PyFrameObject *, int, PyObject *); void uwsgi_python_reset_random_seed(void); + +#ifdef __linux__ +int uwsgi_init_symbol_import(void); +#endif diff --git a/plugins/python/uwsgiplugin.py b/plugins/python/uwsgiplugin.py index 1f7a7b73..4b58db8d 100644 --- a/plugins/python/uwsgiplugin.py +++ b/plugins/python/uwsgiplugin.py @@ -3,7 +3,7 @@ import os,sys from distutils import sysconfig NAME='python' -GCC_LIST = ['python_plugin', 'pyutils', 'pyloader', 'wsgi_handlers', 'wsgi_headers', 'wsgi_subhandler', 'gil', 'uwsgi_pymodule', 'profiler'] +GCC_LIST = ['python_plugin', 'pyutils', 'pyloader', 'wsgi_handlers', 'wsgi_headers', 'wsgi_subhandler', 'gil', 'uwsgi_pymodule', 'profiler', 'symimporter'] CFLAGS = ['-I' + sysconfig.get_python_inc(), '-I' + sysconfig.get_python_inc(plat_specific=True) ] LDFLAGS = [] diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 420300a1..3c12e2b6 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -28,7 +28,7 @@ if not GCC: binary_list = [] def binarize(name): - return name.replace('/', '_').replace('.','_') + return name.replace('/', '_').replace('.','_').replace('-','_') def spcall(cmd): p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=open('/dev/null','w'))