Fix param scoping to use cmake for parsing.

CMakeLists.txt

@@ -273,6 +273,7 @@ if (CATKIN_DEVEL_PREFIX)
 endif()
 
 find_package(PythonInterp REQUIRED)
+px4_find_python_module(jinja2 REQUIRED)
 
 #=============================================================================
 # cmake testing
@@ -401,7 +402,7 @@ px4_generate_messages(TARGET msg_gen
 	)
 px4_generate_parameters_xml(OUT parameters.xml 
 	BOARD ${BOARD}
-	SCOPE ${PX4_SOURCE_DIR}/cmake/configs/${OS}_${BOARD}_${LABEL}.cmake
+	MODULES ${config_module_list}
 	OVERRIDES ${PARAM_DEFAULT_OVERRIDES})
 px4_generate_airframes_xml(OUT airframes.xml BOARD ${BOARD})
 add_custom_target(xml_gen

Tools/px4params/__init__.py

@@ -1 +1 @@
-__all__ = ["srcscanner", "srcparser", "xmlout", "dokuwikiout", "dokuwikirpc", "cmakeparser", "scope"]
+__all__ = ["srcscanner", "srcparser", "xmlout", "dokuwikiout", "dokuwikirpc", "scope"]

Tools/px4params/cmakeparser.py

@@ -1,37 +0,0 @@
-import re
-import codecs
-import sys
-
-class CMakeParser(object):
-    """
-    Parses provided data and stores all found paths in scope.
-    """
-    re_split_lines = re.compile(r'[\r\n]+')
-    re_comment = re.compile(r'^\#')
-    re_start = re.compile(r'set\s*\(\s*config_module_list')
-    re_end = re.compile(r'\)\s*')
-
-    def Parse(self, scope, contents):
-        """
-        Incrementally parse cmake file contents and append all found path scope
-        to scope.
-        """
-        # This code is essentially a comment-parsing grammar. "state"
-        # represents parser state. It contains human-readable state
-        # names.
-        state = None
-        for line in self.re_split_lines.split(contents):
-            line = line.strip()
-            # Ignore empty lines
-            if line == "":
-                continue
-            if self.re_comment.match(line):
-                continue
-            elif self.re_start.match(line):
-                state = "gather"
-                continue
-            elif state is not None and state == "gather":
-                if self.re_end.match(line):
-                    return True
-                scope.Add(line)
-        return False

Tools/px_generate_params.py

@@ -1,107 +1,63 @@
 #!/usr/bin/env python
+"""
+Param source code generation script.
+"""
+from __future__ import print_function
 import xml.etree.ElementTree as ET
-import os
-import re
 import codecs
+import argparse
+from jinja2 import Environment, FileSystemLoader
+import os
 
-from px4params import scope, cmakeparser
+def generate(xml_file, dest='.', modules=None):
+    """
+    Generate px4 param source from xml.
 
-if len(os.sys.argv) < 2:
+    @param xml_file: input parameter xml file
-    print("Error in %s" % os.sys.argv[0])
+    @param dest: Destination directory for generated files
-    print("Usage: %s <parameters.xml> [cmake-file-scoping] " % os.sys.argv[0])
+    @param modules: The list of px4 modules to search for params.
-    raise SystemExit
+        None means to scan everything.
+    """
+    # pylint: disable=broad-except
+    tree = ET.parse(xml_file)
+    root = tree.getroot()
 
-cmake_scope = scope.Scope()
+    params = []
-if len(os.sys.argv) == 3:
+    for group in root:
-	with codecs.open(os.sys.argv[2], 'r', 'utf-8') as f:
+        if group.tag == "group" and "no_code_generation" not in group.attrib:
-		try:
+            for param in group:
-			contents = f.read()
+                scope_ = param.find('scope').text
-			f.close()
+                if (modules is not None) and (not scope_ in modules):
-			parser = cmakeparser.CMakeParser()
+                    continue
-			parser.Parse(cmake_scope, contents)
+                params.append(param)
-		except:
-			contents = ''
-			print('Failed reading file: %s, skipping scoping.' % os.sys.argv[2])
-			pass
 
-fp_header = open("px4_parameters.h", "w")
+    params = sorted(params, key=lambda name: name.attrib["name"])
-fp_src = open("px4_parameters.c", "w")
 
-tree = ET.parse(os.sys.argv[1])
+    script_path = os.path.dirname(os.path.realpath(__file__))
-root = tree.getroot()
 
-# Generate the header file content
+    # for jinja docs see: http://jinja.pocoo.org/docs/2.9/api/
-header = """
+    env = Environment(
-#include <stdint.h>
+        loader=FileSystemLoader(os.path.join(script_path, 'templates')))
-#include <systemlib/param/param.h>
 
-// DO NOT EDIT
+    if not os.path.isdir(dest):
-// This file is autogenerated from parameters.xml
+        os.path.mkdir(dest)
 
-__BEGIN_DECLS
+    template_files = [
+        'px4_parameters.h.jinja',
+        'px4_parameters.c.jinja',
+    ]
+    for template_file in template_files:
+        template = env.get_template(template_file)
+        with open(os.path.join(
+                dest, template_file.replace('.jinja','')), 'w') as fid:
+            fid.write(template.render(params=params))
 
-struct px4_parameters_t {
+if __name__ == "__main__":
-"""
+    arg_parser = argparse.ArgumentParser()
-params = []
+    arg_parser.add_argument("--xml", help="parameter xml file")
-for group in root:
+    arg_parser.add_argument("--modules", help="px4 module list", default=None)
-	if group.tag == "group" and "no_code_generation" not in group.attrib:
+    arg_parser.add_argument("--dest", help="destination path", default=os.path.curdir)
-		for param in group:
+    args = arg_parser.parse_args()
-			scope_ = param.find('scope').text
+    generate(xml_file=args.xml, modules=args.modules, dest=args.dest)
-			if not cmake_scope.Has(scope_):
-				continue
-			params.append(param)
 
-params = sorted(params, key=lambda name: name.attrib["name"])
+#  vim: set et fenc=utf-8 ff=unix sts=4 sw=4 ts=4 : 
-for param in params:
-	header += """
-	const struct param_info_s __param__%s;""" % param.attrib["name"]
-
-header += """
-	const unsigned int param_count;
-};
-
-extern const struct px4_parameters_t px4_parameters;
-"""
-
-# Generate the C file content
-src = """
-#include <px4_parameters.h>
-
-// DO NOT EDIT
-// This file is autogenerated from paramaters.xml
-
-const
-#ifndef __PX4_DARWIN
-__attribute__((used, section("__param")))
-#endif
-struct px4_parameters_t px4_parameters = {
-"""
-i=0
-for param in params:
-	val_str = "#error UNKNOWN PARAM TYPE, FIX px_generate_params.py"
-	if (param.attrib["type"] == "FLOAT"):
-		val_str = ".val.f = "
-	elif (param.attrib["type"] == "INT32"):
-		val_str = ".val.i = "
-	i+=1
-	src += """
-	{
-		"%s",
-		PARAM_TYPE_%s,
-		%s%s
-	},
-""" % (param.attrib["name"], param.attrib["type"], val_str, param.attrib["default"])
-src += """
-	%d
-};
-
-//extern const struct px4_parameters_t px4_parameters;
-
-__END_DECLS
-
-""" % i
-
-fp_header.write(header)
-fp_src.write(src)
-fp_header.close()
-fp_src.close()

Tools/px_process_params.py

@@ -50,7 +50,7 @@ from __future__ import print_function
 import sys
 import os
 import argparse
-from px4params import srcscanner, srcparser, xmlout, dokuwikiout, dokuwikirpc, scope, cmakeparser
+from px4params import srcscanner, srcparser, xmlout, dokuwikiout, dokuwikirpc
 
 import re
 import json
@@ -112,11 +112,12 @@ def main():
                         default="Automagically updated parameter documentation from code.",
                         help="DokuWiki page edit summary")
     parser.add_argument('-v', '--verbose', action='store_true', help="verbose output")
-    parser.add_argument('--scope', default=None, action='store', help="pass the scope (list of compiled modules)")
     parser.add_argument("-o", "--overrides",
                         default="{}",
                         metavar="OVERRIDES",
                         help="a dict of overrides in the form of a json string")
+    parser.add_argument('--modules', default=None, action='store', help="list of compiled modules")
+
 
     args = parser.parse_args()
 
@@ -133,22 +134,8 @@ def main():
     # Scan directories, and parse the files
     if (args.verbose): print("Scanning source path " + args.src_path)
     
-    use_scope = False
+    if args.modules is not None:
-    cmake_scope = scope.Scope();
+        if not scanner.ScanDir([os.path.join(args.src_path, p) for p in args.modules.split(',')], parser):
-    
-    if args.scope:
-        with codecs.open(args.scope, 'r', 'utf-8') as f:
-            try:
-                contents = f.read()
-                f.close()
-                cmake_parser = cmakeparser.CMakeParser()
-                cmake_parser.Parse(cmake_scope, contents)
-                use_scope = True
-            except:
-                use_scope = False
-                pass
-    if use_scope and len(cmake_scope.scope) > 0:
-        if not scanner.ScanDir([os.path.join(args.src_path, p) for p in cmake_scope.scope], parser):
             sys.exit(1)
     else:
         if not scanner.ScanDir([args.src_path], parser):

Tools/templates/px4_parameters.c.jinja

@@ -0,0 +1,33 @@
+{# jinja syntax: http://jinja.pocoo.org/docs/2.9/templates/ #}
+#include <px4_parameters.h>
+
+// DO NOT EDIT
+// This file is autogenerated from paramaters.xml
+
+__BEGIN_DECLS
+
+const
+#ifndef __PX4_DARWIN
+__attribute__((used, section("__param")))
+#endif
+
+struct px4_parameters_t px4_parameters = {
+{% for param in params %}
+	{
+		"{{ param.attrib["name"] }}",
+		PARAM_TYPE_{{ param.attrib["type"] }},
+	{%- if param.attrib["type"] == "FLOAT" %}
+		.val.f = {{ param.attrib["default"] }}
+	{%- elif param.attrib["type"] == "INT32" %}
+		.val.i = {{ param.attrib["default"] }}
+	{%- endif %}
+	},
+{% endfor %}
+	{{ params | length }}
+};
+
+//extern const struct px4_parameters_t px4_parameters;
+
+__END_DECLS
+
+{# vim: set noet ft=jinja fenc=utf-8 ff=unix sts=4 sw=4 ts=4 : #}

Tools/templates/px4_parameters.h.jinja

@@ -0,0 +1,21 @@
+{# jinja syntax: http://jinja.pocoo.org/docs/2.9/templates/ #}
+#include <stdint.h>
+#include <systemlib/param/param.h>
+
+// DO NOT EDIT
+// This file is autogenerated from parameters.xml
+
+__BEGIN_DECLS
+
+struct px4_parameters_t {
+{%- for param in params %}
+	const struct param_info_s __param__{{ param.attrib["name"] }};
+{%- endfor %}
+	const unsigned int param_count;
+};
+
+extern const struct px4_parameters_t px4_parameters;
+
+__END_DECLS
+
+{# vim: set noet ft=jinja fenc=utf-8 ff=unix sts=4 sw=4 ts=4 : #}

cmake/common/px4_base.cmake

@@ -1008,6 +1008,9 @@ endfunction()
 #
 #	Input:
 #		BOARD : the board
+#		MODULES : a list of px4 modules used to limit scope of the paramaters
+#		OVERRIDES : A json dict with param names as keys and param default
+# 			overrides as values
 #
 #	Output:
 #		OUT	: the generated xml file
@@ -1018,8 +1021,9 @@ endfunction()
 function(px4_generate_parameters_xml)
 	px4_parse_function_args(
 		NAME px4_generate_parameters_xml
-		ONE_VALUE OUT BOARD SCOPE OVERRIDES
+		ONE_VALUE OUT BOARD OVERRIDES
-		REQUIRED OUT BOARD
+		MULTI_VALUE MODULES
+		REQUIRED MODULES OUT BOARD
 		ARGN ${ARGN})
 	set(path ${PX4_SOURCE_DIR}/src)
 	file(GLOB_RECURSE param_src_files
@@ -1028,10 +1032,11 @@ function(px4_generate_parameters_xml)
 	if (NOT OVERRIDES)
 		set(OVERRIDES "{}")
 	endif()
+	px4_join(OUT module_list  LIST ${MODULES} GLUE ",")
 	add_custom_command(OUTPUT ${OUT}
 		COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/px_process_params.py
-			-s ${path} --board CONFIG_ARCH_${BOARD} --xml --inject-xml --scope ${SCOPE}
+			-s ${path} --board CONFIG_ARCH_${BOARD} --xml --inject-xml
-			--overrides ${OVERRIDES}
+			--overrides ${OVERRIDES} --modules ${module_list}
 		DEPENDS ${param_src_files} ${PX4_SOURCE_DIR}/Tools/px_process_params.py
 			${PX4_SOURCE_DIR}/Tools/px_generate_params.py
 		)
@@ -1045,36 +1050,36 @@ endfunction()
 #	Generates a source file with all parameters.
 #
 #	Usage:
-#		px4_generate_parameters_source(OUT <list-source-files> XML <param-xml-file> [SCOPE <cmake file for scoping>])
+#		px4_generate_parameters_source(OUT <list-source-files> XML <param-xml-file> MODULES px4 module list)
 #
 #	Input:
 #		XML   : the parameters.xml file
-#		SCOPE : the cmake file used to limit scope of the paramaters
+#		MODULES : a list of px4 modules used to limit scope of the paramaters
 #		DEPS  : target dependencies
 #
 #	Output:
 #		OUT	: the generated source files
 #
 #	Example:
-#		px4_generate_parameters_source(OUT param_files XML parameters.xml SCOPE ${OS}_${BOARD}_${LABEL}.cmake )
+#		px4_generate_parameters_source(OUT param_files XML parameters.xml MODULES lib/controllib modules/ekf2)
 #
 function(px4_generate_parameters_source)
 	px4_parse_function_args(
 		NAME px4_generate_parameters_source
-		ONE_VALUE OUT XML SCOPE DEPS
+		ONE_VALUE OUT XML DEPS
-		REQUIRED OUT XML
+		MULTI_VALUE MODULES
+		REQUIRED MODULES OUT XML
 		ARGN ${ARGN})
 	set(generated_files
 		${CMAKE_CURRENT_BINARY_DIR}/px4_parameters.h
 		${CMAKE_CURRENT_BINARY_DIR}/px4_parameters.c)
 	set_source_files_properties(${generated_files}
 		PROPERTIES GENERATED TRUE)
-	if ("${config_generate_parameters_scope}" STREQUAL "ALL")
+	px4_join(OUT module_list  LIST ${MODULES} GLUE ",")
-		set(SCOPE "")
-	endif()
 	add_custom_command(OUTPUT ${generated_files}
-		COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/px_generate_params.py ${XML} ${SCOPE}
+		COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/px_generate_params.py
-		DEPENDS ${XML} ${DEPS} ${SCOPE}
+		--xml ${XML} --modules ${module_list} --dest ${CMAKE_CURRENT_BINARY_DIR}
+		DEPENDS ${XML} ${DEPS}
 		)
 	set(${OUT} ${generated_files} PARENT_SCOPE)
 endfunction()
@@ -1259,4 +1264,43 @@ function(px4_add_library target)
 	set(_no_optimization_for_target ${_no_optimization_for_target} PARENT_SCOPE)
 endfunction()
 
+#=============================================================================
+#
+#	px4_find_python_module
+#
+#	Find a required python module
+#
+#   Usage
+#		px4_find_python_module(module_name [REQUIRED])
+#
+function(px4_find_python_module module)
+	string(TOUPPER ${module} module_upper)
+	if(NOT PY_${module_upper})
+		if(ARGC GREATER 1 AND ARGV1 STREQUAL "REQUIRED")
+			set(PY_${module}_FIND_REQUIRED TRUE)
+		endif()
+		# A module's location is usually a directory, but for binary modules
+		# it's a .so file.
+		execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
+			"import re, ${module}; print(re.compile('/__init__.py.*').sub('',${module}.__file__))"
+			RESULT_VARIABLE _${module}_status
+			OUTPUT_VARIABLE _${module}_location
+			ERROR_QUIET 
+			OUTPUT_STRIP_TRAILING_WHITESPACE)
+		if(NOT _${module}_status)
+			set(PY_${module_upper} ${_${module}_location} CACHE STRING
+				"Location of Python module ${module}")
+		endif()
+	endif()
+	find_package_handle_standard_args(PY_${module}
+		"couldn't find python module ${module}:
+		\nfor debian systems try: \
+		\n\tsudo apt-get install python-${module} \
+		\nor for all other OSs/debian: \
+		\n\tpip install ${module}\n" PY_${module_upper})
+	#if (NOT PY_${module}_FOUND)
+		#message(FATAL_ERROR "python module not found, exitting")
+	#endif()
+endfunction(px4_find_python_module)
+
 # vim: set noet fenc=utf-8 ff=unix nowrap:

src/modules/param/CMakeLists.txt

@@ -35,7 +35,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 px4_generate_parameters_source(OUT param_files
 	XML ${PX4_BINARY_DIR}/parameters.xml
-	SCOPE ${PX4_SOURCE_DIR}/cmake/configs/${OS}_${BOARD}_${LABEL}.cmake
+	MODULES ${config_module_list}
 	DEPS xml_gen
 	)