first working prototype of embedded plugins builder

core/plugins_builder.c

@@ -8,64 +8,106 @@
 
 		mkdir(.uwsgi_plugin_builder)
 		generate .uwsgi_plugin_builder/uwsgi.h
-		setenv(CFLAGS=uwsgi_cflags)
+		generate .uwsgi_plugin_builder/uwsgiconfig.py
-		pipe uwsgiconfig.py to python passing args
+		setenv(UWSGI_PLUGINS_BUILDER_CFLAGS=uwsgi_cflags)
+		exec PYTHON .uwsgi_plugin_builder/uwsgiconfig.py --extra-plugin <directory> [name]
 
 */
 
-int uwsgi_build_plugin(char *directory) {
+void uwsgi_build_plugin(char *directory) {
 	uwsgi_log("building uWSGI plugin %s...\n", directory);
 
 	if (!uwsgi_file_exists(UWSGI_BUILD_DIR)) {
 		if (mkdir(UWSGI_BUILD_DIR, S_IRWXU) < 0) {
         		uwsgi_error("uwsgi_build_plugin()/mkdir() " UWSGI_BUILD_DIR "/");
-			return 1;
+			_exit(1);
 		}
 	}
 
 	char *dot_h = uwsgi_get_dot_h();
 	if (!dot_h) {
 		uwsgi_log("unable to generate uwsgi.h");
-		return 1;
+		_exit(1);
 	}
 
 	if (strlen(dot_h) == 0) {
 		free(dot_h);
 		uwsgi_log("invalid uwsgi.h");
-		return 1;
+		_exit(1);
 	}
 
 	int dot_h_fd = open(UWSGI_BUILD_DIR "/uwsgi.h", O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR);
 	if (dot_h_fd < 0) {
 		uwsgi_error_open(UWSGI_BUILD_DIR "/uwsgi.h");
 		free(dot_h);
-		return 1;
+		_exit(1);
 	}
 
 	ssize_t dot_h_len = (ssize_t) strlen(dot_h);
 	if (write(dot_h_fd, dot_h, dot_h_len) != dot_h_len) {
 		uwsgi_error("uwsgi_build_plugin()/write()");
-		close(dot_h_fd);
+		_exit(1);
-		free(dot_h);
+	}
-		return 1;
+
+	char *config_py = uwsgi_get_config_py();
+        if (!config_py) {
+                uwsgi_log("unable to generate uwsgiconfig.py");
+                _exit(1);
+        }
+
+        if (strlen(config_py) == 0) {
+                uwsgi_log("invalid uwsgiconfig.py");
+                _exit(1);
+        }
+
+        int config_py_fd = open(UWSGI_BUILD_DIR "/uwsgiconfig.py", O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR);
+        if (config_py_fd < 0) {
+                uwsgi_error_open(UWSGI_BUILD_DIR "/uwsgiconfig.py");
+                _exit(1);
+        }
+
+        ssize_t config_py_len = (ssize_t) strlen(config_py);
+        if (write(config_py_fd, config_py, config_py_len) != config_py_len) {
+                uwsgi_error("uwsgi_build_plugin()/write()");
+                _exit(1);
+        }
+
+	char *cflags = uwsgi_get_cflags();
+	if (!cflags) {
+		uwsgi_log("unable to find cflags\n");
+		_exit(1);
+	}
+	if (strlen(cflags) == 0) {
+		uwsgi_log("invalid cflags\n");
+		_exit(1);
+	}
+
+	if (setenv("UWSGI_PLUGINS_BUILDER_CFLAGS", cflags, 1)) {
+		uwsgi_error("uwsgi_build_plugin()/setenv()");
+		_exit(1);
 	}
 	
-	free(dot_h);
+	// now run the python script
-	close(dot_h_fd);
+	char *argv[6];
-
-	// now extract the python build script from the process
-	// and pipe it to python
-
-	char *argv[4];
 
 	argv[0] = getenv("PYTHON");
 	if (!argv[0]) argv[0] = "python";
 
-	argv[1] = "-";
+	argv[1] = UWSGI_BUILD_DIR "/uwsgiconfig.py";
 	argv[2] = "--extra-plugin";
-	argv[3] = directory;
+	char *space = strchr(directory, ' ');
+	if (space) {
+		*space = 0;
+		argv[3] = directory;
+                argv[4] = space+1;
+		argv[5] = NULL;
+	}
+	else {
+		argv[3] = directory;
+		argv[4] = NULL;
+	}
 
 	execvp(argv[0], argv);
 	// never here...	
-	return 1;
+	_exit(1);
 }

core/uwsgi.c

@@ -907,6 +907,7 @@ static struct uwsgi_option uwsgi_base_options[] = {
 	{"exit", optional_argument, 0, "force exit() of the instance", uwsgi_opt_exit, NULL, UWSGI_OPT_IMMEDIATE},
 	{"cflags", no_argument, 0, "report uWSGI CFLAGS (useful for building external plugins)", uwsgi_opt_cflags, NULL, UWSGI_OPT_IMMEDIATE},
 	{"dot-h", no_argument, 0, "dump the uwsgi.h used for building the core  (useful for building external plugins)", uwsgi_opt_dot_h, NULL, UWSGI_OPT_IMMEDIATE},
+	{"config-py", no_argument, 0, "dump the uwsgiconfig.py used for building the core  (useful for building external plugins)", uwsgi_opt_config_py, NULL, UWSGI_OPT_IMMEDIATE},
 	{"build-plugin", required_argument, 0, "build a uWSGI plugin for the current binary", uwsgi_opt_build_plugin, NULL, UWSGI_OPT_IMMEDIATE},
 	{"version", no_argument, 0, "print uWSGI version", uwsgi_opt_print, UWSGI_VERSION, 0},
 	{0, 0, 0, 0, 0, 0, 0}
@@ -4507,8 +4508,44 @@ void uwsgi_opt_dot_h(char *opt, char *filename, void *foobar) {
         exit(0);
 }
 
+extern char *uwsgi_config_py;
+char *uwsgi_get_config_py() {
+        char *src = uwsgi_config_py;
+        size_t len = strlen(src);
+        char *ptr = uwsgi_malloc(len / 2);
+        char *base = ptr;
+        size_t i;
+        unsigned int u;
+        for (i = 0; i < len; i += 2) {
+                sscanf(src + i, "%2x", &u);
+                *ptr++ = (char) u;
+        }
+#ifdef UWSGI_ZLIB
+        struct uwsgi_buffer *ub = uwsgi_zlib_decompress(base, ptr-base);
+        if (!ub) {
+                free(base);
+                return "";
+        }
+        // add final null byte
+        uwsgi_buffer_append(ub, "\0", 1);
+        free(base);
+        // base is the final blob
+        base = ub->buf;
+        ub->buf = NULL;
+        uwsgi_buffer_destroy(ub);
+#endif
+        return base;
+}
+
+void uwsgi_opt_config_py(char *opt, char *filename, void *foobar) {
+        fprintf(stdout, "%s\n", uwsgi_get_config_py());
+        exit(0);
+}
+
+
 void uwsgi_opt_build_plugin(char *opt, char *directory, void *foobar) {
-	_exit(uwsgi_build_plugin(directory));
+	uwsgi_build_plugin(directory);
+	exit(1);
 }
 
 void uwsgi_opt_connect_and_read(char *opt, char *address, void *foobar) {

uwsgi.h

@@ -3763,10 +3763,12 @@ void uwsgi_opt_add_custom_option(char *, char *, void *);
 void uwsgi_opt_cflags(char *, char *, void *);
 void uwsgi_opt_build_plugin(char *, char *, void *);
 void uwsgi_opt_dot_h(char *, char *, void *);
+void uwsgi_opt_config_py(char *, char *, void *);
 void uwsgi_opt_connect_and_read(char *, char *, void *);
 void uwsgi_opt_extract(char *, char *, void *);
 
 char *uwsgi_get_dot_h();
+char *uwsgi_get_config_py();
 char *uwsgi_get_cflags();
 
 struct uwsgi_string_list *uwsgi_string_list_has_item(struct uwsgi_string_list *, char *, size_t);
@@ -4529,7 +4531,7 @@ struct uwsgi_protocol *uwsgi_register_protocol(char *, void (*)(struct uwsgi_soc
 
 void uwsgi_protocols_register(void);
 
-int uwsgi_build_plugin(char *dir);
+void uwsgi_build_plugin(char *dir);
 
 #ifdef __cplusplus
 }

uwsgiconfig.py

@@ -305,6 +305,24 @@ def build_uwsgi(uc, print_only=False, gcll=None):
         uwsgi_dot_h = uwsgi_dot_h_content.encode('hex')
     open('core/dot_h.c', 'w').write('char *uwsgi_dot_h = "%s";\n' % uwsgi_dot_h);
     gcc_list.append('core/dot_h') 
+
+    # embed uwsgiconfig.py in the server binary. It increases the binary size, but will be very useful
+    # if possibile, the blob is compressed
+    if sys.version_info[0] >= 3:
+        uwsgi_config_py_content = open('uwsgiconfig.py', 'rb').read()
+    else:
+        uwsgi_config_py_content = open('uwsgiconfig.py').read()
+    if report['zlib']:
+        import zlib
+        # maximum level of compression
+        uwsgi_config_py_content = zlib.compress(uwsgi_config_py_content, 9)
+    if sys.version_info[0] >= 3:
+        import binascii
+        uwsgi_config_py = binascii.b2a_hex(uwsgi_config_py_content).decode('ascii')
+    else:
+        uwsgi_config_py = uwsgi_config_py_content.encode('hex')
+    open('core/config_py.c', 'w').write('char *uwsgi_config_py = "%s";\n' % uwsgi_config_py);
+    gcc_list.append('core/config_py')
     
     cflags.append('-DUWSGI_CFLAGS=\\"%s\\"' % uwsgi_cflags)
     cflags.append('-DUWSGI_BUILD_DATE="\\"%s\\""' % time.strftime("%d %B %Y %H:%M:%S"))
@@ -1363,7 +1381,7 @@ if __name__ == "__main__":
     parser.add_option("-f", "--cflags", action="callback", callback=vararg_callback, dest="cflags", help="same as --build but less verbose", metavar="PROFILE")
     parser.add_option("-u", "--unbit", action="store_true", dest="unbit", help="build unbit profile")
     parser.add_option("-p", "--plugin", action="callback", callback=vararg_callback, dest="plugin", help="build a plugin as shared library, optionally takes a build profile name", metavar="PLUGIN [PROFILE]")
-    parser.add_option("-x", "--extra-plugin", action="callback", callback=vararg_callback,  dest="extra_plugin", help="build an external plugin as shared library, takes an optional include dir", metavar="PLUGIN [INCLUDE_DIR]")
+    parser.add_option("-x", "--extra-plugin", action="callback", callback=vararg_callback,  dest="extra_plugin", help="build an external plugin as shared library, takes an optional include dir", metavar="PLUGIN [NAME]")
     parser.add_option("-c", "--clean", action="store_true", dest="clean", help="clean the build")
     parser.add_option("-e", "--check", action="store_true", dest="check", help="run cppcheck")
     parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="more verbose build")
@@ -1434,7 +1452,12 @@ if __name__ == "__main__":
         print("*** uWSGI building and linking plugin %s ***" % options.extra_plugin[0])
         cflags = os.environ['UWSGI_PLUGINS_BUILDER_CFLAGS'].split()
         cflags.append('-I.uwsgi_plugins_builder/')
-        build_plugin(options.extra_plugin[0], None, cflags, [], [], None)
+        name = None
+        try:
+            name = options.extra_plugin[1]
+        except:
+            pass
+        build_plugin(options.extra_plugin[0], None, cflags, [], [], name)
     elif options.clean:
         os.system("rm -f core/*.o")
         os.system("rm -f proto/*.o")
@@ -1442,6 +1465,7 @@ if __name__ == "__main__":
         os.system("rm -f plugins/*/*.o")
         os.system("rm -f build/*.o")
         os.system("rm -f core/dot_h.c")
+        os.system("rm -f core/config_py.c")
     elif options.check:
         os.system("cppcheck --max-configs=1000 --enable=all -q core/ plugins/ proto/ lib/ apache2/")
     else: