mavlink: implement COMPONENT_INFORMATION

- implement COMP_METADATA_TYPE_VERSION & add component_version.json.gz to
  ROMFS for all targets.
- implement COMP_METADATA_TYPE_PARAMETER (only enabled for SITL currently)

cmake/metadata.cmake

@@ -59,7 +59,8 @@ add_custom_target(metadata_parameters
 	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/src/lib/parameters/px_process_params.py
 		--src-path `find ${PX4_SOURCE_DIR}/src -maxdepth 4 -type d`
 		--inject-xml ${PX4_SOURCE_DIR}/src/lib/parameters/parameters_injected.xml
-		--json ${PX4_BINARY_DIR}/docs/parameters.json
+		--json ${PX4_BINARY_DIR}/docs/params.json
+		--compress
 
 	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/src/lib/parameters/px_process_params.py
 		--src-path `find ${PX4_SOURCE_DIR}/src -maxdepth 4 -type d`

cmake/px4_add_board.cmake

@@ -209,12 +209,14 @@ function(px4_add_board)
 	endif()
 	foreach(metadata ${EMBEDDED_METADATA})
 		if(${metadata} STREQUAL "parameters")
-			list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/parameters.json.gz)
+			list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/params.json.gz)
 			list(APPEND romfs_extra_dependencies parameters_xml)
 		else()
 			message(FATAL_ERROR "invalid value for EMBEDDED_METADATA: ${metadata}")
 		endif()
 	endforeach()
+	list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/component_version.json.gz)
+	list(APPEND romfs_extra_dependencies component_version_json)
 	set(config_romfs_extra_files ${romfs_extra_files} CACHE INTERNAL "extra ROMFS files" FORCE)
 	set(config_romfs_extra_dependencies ${romfs_extra_dependencies} CACHE INTERNAL "extra ROMFS deps" FORCE)
 

src/lib/CMakeLists.txt

@@ -43,6 +43,7 @@ add_subdirectory(cdev)
 add_subdirectory(circuit_breaker)
 add_subdirectory(collision_prevention)
 add_subdirectory(controllib)
+add_subdirectory(component_information)
 add_subdirectory(conversion)
 add_subdirectory(drivers)
 add_subdirectory(ecl)

src/lib/component_information/CMakeLists.txt

@@ -0,0 +1,70 @@
+############################################################################
+#
+#   Copyright (c) 2020 PX4 Development Team. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name PX4 nor the names of its contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+############################################################################
+
+set(component_version_extra_args)
+# We disable parameter support if the metadata is not in the ROMFS. This can be
+# removed once it is stored on a server
+list(FIND config_romfs_extra_dependencies "parameters_xml" index)
+if (${index} EQUAL -1)
+	set(component_version_extra_args "--no-parameters")
+endif()
+
+set(component_version_json ${PX4_BINARY_DIR}/component_version.json)
+add_custom_command(OUTPUT ${component_version_json} ${component_version_json}.gz
+	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_component_version.py
+		${component_version_json}
+		--compress
+		${component_version_extra_args}
+	DEPENDS
+		generate_component_version.py
+	COMMENT "Generating component_version.json"
+)
+add_custom_target(component_version_json DEPENDS ${component_version_json})
+
+
+set(component_information_header ${CMAKE_CURRENT_BINARY_DIR}/hashes.h)
+add_custom_command(OUTPUT ${component_information_header}
+	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_hashes.py
+		${PX4_BINARY_DIR}/params.json.gz
+		${component_version_json}
+		--output ${component_information_header}
+	DEPENDS
+		generate_hashes.py
+		parameters_xml
+		${PX4_BINARY_DIR}/params.json.gz
+		${component_version_json}
+	COMMENT "Generating component_information/hashes.h"
+)
+add_custom_target(component_information_header DEPENDS ${component_information_header})
+
+

src/lib/component_information/generate_component_version.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import gzip
+
+parser = argparse.ArgumentParser(description="""Generate the COMPONENT_VERSION json file""")
+parser.add_argument('filename', metavar='component_version.json', help='JSON output file')
+parser.add_argument('--compress', action='store_true', help='Add a compressed output file')
+parser.add_argument('--no-parameters', action='store_true', help='disable parameter support')
+
+args = parser.parse_args()
+filename = args.filename
+compress = args.compress
+
+def save_compressed(filename):
+    #create gz compressed version
+    gz_filename=filename+'.gz'
+    with gzip.open(gz_filename, 'wt') as f:
+        with open(filename, 'r') as content_file:
+            f.write(content_file.read())
+
+component_version = {}
+component_version['version'] = 1
+# types:
+# 0: COMP_METADATA_TYPE_VERSION
+# 1: COMP_METADATA_TYPE_PARAMETER
+# 2: COMP_METADATA_TYPE_COMMANDS
+supported_types = [0]
+if not args.no_parameters: supported_types.append(1)
+component_version['supportedCompMetadataTypes'] = supported_types
+
+with open(filename, 'w') as outfile:
+    json.dump(component_version, outfile)
+
+if compress:
+    save_compressed(filename)

src/lib/component_information/generate_hashes.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+
+import argparse
+import zlib
+import os
+
+parser = argparse.ArgumentParser(description="""Generate the COMPONENT_INFORMATION hashes header file""")
+parser.add_argument('--output', metavar='hashes.h', help='output file')
+parser.add_argument('files', nargs='+', metavar='hashes.json', help='files to generate hashes from')
+
+args = parser.parse_args()
+filename = args.output
+files = args.files
+
+with open(filename, 'w') as outfile:
+    outfile.write("#include <stdint.h>\n")
+    outfile.write("namespace component_information {\n")
+    for filename in files:
+        # get CRC
+        file_hash = 0
+        for line in open(filename, "rb"):
+            file_hash = zlib.crc32(line, file_hash)
+
+        basename = os.path.basename(filename)
+        identifier = basename.split('.')[0]
+        outfile.write("static constexpr uint32_t {:}_hash = {:};\n".format(identifier, file_hash))
+
+
+    outfile.write("}\n")
+
+

src/lib/parameters/CMakeLists.txt

@@ -87,7 +87,7 @@ add_custom_command(OUTPUT ${generated_serial_params_file}
 )
 
 set(parameters_xml ${PX4_BINARY_DIR}/parameters.xml)
-set(parameters_json ${PX4_BINARY_DIR}/parameters.json)
+set(parameters_json ${PX4_BINARY_DIR}/params.json) # file name needs to be kept short to fit into url (metadata_uri)
 file(GLOB_RECURSE param_src_files ${PX4_SOURCE_DIR}/src/*params.c)
 add_custom_command(OUTPUT ${parameters_xml} ${parameters_json} ${parameters_json}.gz
 	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/px_process_params.py

src/modules/mavlink/CMakeLists.txt

@@ -64,6 +64,7 @@ px4_add_module(
 		module.yaml
 	DEPENDS
 		airspeed
+		component_information_header
 		drivers_accelerometer
 		drivers_barometer
 		drivers_gyroscope

src/modules/mavlink/mavlink_messages.cpp

@@ -104,6 +104,7 @@ using matrix::wrap_2pi;
 #include "streams/ATTITUDE_TARGET.hpp"
 #include "streams/AUTOPILOT_VERSION.hpp"
 #include "streams/COLLISION.hpp"
+#include "streams/COMPONENT_INFORMATION.hpp"
 #include "streams/DISTANCE_SENSOR.hpp"
 #include "streams/ESC_INFO.hpp"
 #include "streams/ESC_STATUS.hpp"
@@ -3151,6 +3152,9 @@ static const StreamListItem streams_list[] = {
 #if defined(STORAGE_INFORMATION_HPP)
 	create_stream_list_item<MavlinkStreamStorageInformation>(),
 #endif // STORAGE_INFORMATION_HPP
+#if defined(COMPONENT_INFORMATION_HPP)
+	create_stream_list_item<MavlinkStreamComponentInformation>(),
+#endif // COMPONENT_INFORMATION_HPP
 #if defined(RAW_RPM_HPP)
 	create_stream_list_item<MavlinkStreamRawRpm>()
 #endif // RAW_RPM_HPP

src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp

@@ -0,0 +1,129 @@
+/****************************************************************************
+ *
+ *   Copyright (c) 2020 PX4 Development Team. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name PX4 nor the names of its contributors may be
+ *    used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ ****************************************************************************/
+
+#ifndef COMPONENT_INFORMATION_HPP
+#define COMPONENT_INFORMATION_HPP
+
+#include "../mavlink_stream.h"
+
+#include <component_information/hashes.h>
+
+#include <px4_platform_common/defines.h>
+
+#include <sys/stat.h>
+
+class MavlinkStreamComponentInformation : public MavlinkStream
+{
+public:
+	static MavlinkStream *new_instance(Mavlink *mavlink) { return new MavlinkStreamComponentInformation(mavlink); }
+
+	static constexpr const char *get_name_static() { return "COMPONENT_INFORMATION"; }
+	static constexpr uint16_t get_id_static() { return MAVLINK_MSG_ID_COMPONENT_INFORMATION; }
+
+	const char *get_name() const override { return get_name_static(); }
+	uint16_t get_id() override { return get_id_static(); }
+
+	unsigned get_size() override
+	{
+		return 0; // never streamed
+	}
+
+	bool request_message(float param2, float param3, float param4,
+			     float param5, float param6, float param7) override
+	{
+		int type = (int)roundf(param2);
+		mavlink_component_information_t component_info{};
+		bool handled = false;
+		PX4_DEBUG("COMPONENT_INFORMATION request type %i", type);
+
+		switch (type) {
+		case COMP_METADATA_TYPE_VERSION:
+			component_info.metadata_uid = component_information::component_version_hash;
+			handled = get_component_information("component_version.json.gz", component_info);
+			break;
+
+		case COMP_METADATA_TYPE_PARAMETER:
+			component_info.metadata_uid = component_information::params_hash;
+			handled = get_component_information("params.json.gz", component_info);
+			break;
+
+		case COMP_METADATA_TYPE_COMMANDS:
+			// TODO
+			break;
+		}
+
+		if (handled) {
+			component_info.metadata_type = type;
+			component_info.time_boot_ms = hrt_absolute_time() / 1000;
+			mavlink_msg_component_information_send_struct(_mavlink->get_channel(), &component_info);
+		}
+
+		return handled;
+	}
+private:
+	explicit MavlinkStreamComponentInformation(Mavlink *mavlink) : MavlinkStream(mavlink) {}
+
+	bool send() override
+	{
+		return false;
+	}
+
+	bool get_component_information(const char *file, mavlink_component_information_t &component_info)
+	{
+		char full_path[64];
+		snprintf(full_path, sizeof(full_path), "%s/etc/extras/%s", PX4_ROOTFSDIR, file);
+		full_path[sizeof(full_path) - 1] = '\0';
+
+		if (file_exist(full_path)) {
+			if (snprintf(component_info.metadata_uri, sizeof(component_info.metadata_uri), "mftp://etc/extras/%s", file)
+			    >= (int)sizeof(component_info.metadata_uri)) {
+				PX4_ERR("path too long (%s)", file);
+				return false;
+			}
+
+		} else {
+			// TODO: use server uri
+			return false;
+		}
+
+		return true;
+	}
+
+	bool file_exist(const char *filename)
+	{
+		struct stat buffer;
+		return stat(filename, &buffer) == 0;
+	}
+};
+
+#endif // COMPONENT_INFORMATION_HPP