From b4add65ed70ef773c20353b952cba917ce3afb07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:27:07 +0300 Subject: [PATCH 01/11] Wrapper for adding environment variable arguments Environment variable name is passed to "metavar" in argparse, so it is now printed in the help text. Formatter class is switched to RawDescriptionHelpFormatter to make it possible to write more detailed group level descriptions in the future. The default value or choices for an argument is printed by the new wrapper instead of the formatter. --- .../wirepas_gateway/utils/argument_tools.py | 209 ++++++++++++------ 1 file changed, 147 insertions(+), 62 deletions(-) diff --git a/python_transport/wirepas_gateway/utils/argument_tools.py b/python_transport/wirepas_gateway/utils/argument_tools.py index e2b05442..24975d70 100644 --- a/python_transport/wirepas_gateway/utils/argument_tools.py +++ b/python_transport/wirepas_gateway/utils/argument_tools.py @@ -70,7 +70,7 @@ class ParserHelper: def __init__( self, description="argument parser", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, + formatter_class=argparse.RawDescriptionHelpFormatter, version=None, ): super(ParserHelper, self).__init__() @@ -195,53 +195,90 @@ def str2none(value): return None return value + def add_env_argument(self, + group, + env_variable, + *args, + **kwargs): + """ + Wrapper for adding an argument which can be set with an environment + variable. Arbitrary arguments are passed to argparse. + """ + default = kwargs.get("default") + kwargs["default"] = os.environ.get(env_variable, default) + + help_text = [] + if "help" in kwargs: + help_text.append(kwargs["help"]) + if "choices" in kwargs: + choice_strs = [str(choice) for choice in kwargs["choices"]] + result = '[%s]' % ', '.join(choice_strs) + help_text.append(f"(choices: {result})") + help_text.append(f"(default: {default})") + + kwargs["help"] = " ".join(help_text) + kwargs["metavar"] = "$" + env_variable + group.add_argument(*args, **kwargs) + def add_file_settings(self): """ For file setting handling""" - self.file_settings.add_argument( + self.add_env_argument( + self.file_settings, + "WM_GW_FILE_SETTINGS", "--settings", type=self.str2none, required=False, - default=os.environ.get("WM_GW_FILE_SETTINGS", None), + default=None, help="A yaml file with argument parameters (see help for options).", ) def add_mqtt(self): """ Commonly used MQTT arguments """ - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_HOSTNAME", "--mqtt_hostname", - default=os.environ.get("WM_SERVICES_MQTT_HOSTNAME", None), + default=None, action="store", type=self.str2none, help="MQTT broker hostname.", ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_USERNAME", "--mqtt_username", - default=os.environ.get("WM_SERVICES_MQTT_USERNAME", None), + default=None, action="store", type=self.str2none, help="MQTT broker username.", ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_PASSWORD", "--mqtt_password", - default=os.environ.get("WM_SERVICES_MQTT_PASSWORD", None), + default=None, action="store", type=self.str2none, help="MQTT broker password.", ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_PORT", "--mqtt_port", - default=os.environ.get("WM_SERVICES_MQTT_PORT", 8883), + default=8883, action="store", type=self.str2int, help="MQTT broker port.", ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_CA_CERTS", "--mqtt_ca_certs", - default=os.environ.get("WM_SERVICES_MQTT_CA_CERTS", None), + default=None, action="store", type=self.str2none, help=( @@ -252,17 +289,21 @@ def add_mqtt(self): ), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_CLIENT_CRT", "--mqtt_certfile", - default=os.environ.get("WM_SERVICES_MQTT_CLIENT_CRT", None), + default=None, action="store", type=self.str2none, help=("Path to the PEM encoded client certificate."), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_CLIENT_KEY", "--mqtt_keyfile", - default=os.environ.get("WM_SERVICES_MQTT_CLIENT_KEY", None), + default=None, action="store", type=self.str2none, help=( @@ -272,9 +313,11 @@ def add_mqtt(self): ), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_CERT_REQS", "--mqtt_cert_reqs", - default=os.environ.get("WM_SERVICES_MQTT_CERT_REQS", "CERT_REQUIRED"), + default="CERT_REQUIRED", choices=["CERT_REQUIRED", "CERT_OPTIONAL", "CERT_NONE"], action="store", type=self.str2none, @@ -285,9 +328,11 @@ def add_mqtt(self): ), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_TLS_VERSION", "--mqtt_tls_version", - default=os.environ.get("WM_SERVICES_MQTT_TLS_VERSION", "PROTOCOL_TLSv1_2"), + default="PROTOCOL_TLSv1_2", choices=[ "PROTOCOL_TLS", "PROTOCOL_TLS_CLIENT", @@ -301,9 +346,11 @@ def add_mqtt(self): help=("Specifies the version of the SSL / TLS protocol to be used."), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_CIPHERS", "--mqtt_ciphers", - default=os.environ.get("WM_SERVICES_MQTT_CIPHERS", None), + default=None, action="store", type=self.str2none, help=( @@ -313,9 +360,11 @@ def add_mqtt(self): ), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_PERSIST_SESSION", "--mqtt_persist_session", - default=os.environ.get("WM_SERVICES_MQTT_PERSIST_SESSION", False), + default=False, type=self.str2bool, nargs="?", const=True, @@ -325,27 +374,33 @@ def add_mqtt(self): ), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_FORCE_UNSECURE", "--mqtt_force_unsecure", - default=os.environ.get("WM_SERVICES_MQTT_FORCE_UNSECURE", False), + default=False, type=self.str2bool, nargs="?", const=True, help=("When True the broker will skip the TLS handshake."), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_ALLOW_UNTRUSTED", "--mqtt_allow_untrusted", - default=os.environ.get("WM_SERVICES_MQTT_ALLOW_UNTRUSTED", False), + default=False, type=self.str2bool, nargs="?", const=True, help=("When true the client will skip the certificate name check."), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_RECONNECT_DELAY", "--mqtt_reconnect_delay", - default=os.environ.get("WM_SERVICES_MQTT_RECONNECT_DELAY", 0), + default=0, action="store", type=self.str2int, help=( @@ -354,17 +409,21 @@ def add_mqtt(self): ), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_MAX_INFLIGHT_MESSAGES", "--mqtt_max_inflight_messages", - default=os.environ.get("WM_SERVICES_MQTT_MAX_INFLIGHT_MESSAGES", 20), + default=20, action="store", type=self.str2int, help=("Max inflight messages for messages with qos > 0"), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_USE_WEBSOCKET", "--mqtt_use_websocket", - default=os.environ.get("WM_SERVICES_MQTT_USE_WEBSOCKET", False), + default=False, type=self.str2bool, nargs="?", const=True, @@ -373,9 +432,11 @@ def add_mqtt(self): ), ) - self.mqtt.add_argument( + self.add_env_argument( + self.mqtt, + "WM_SERVICES_MQTT_RATE_LIMIT_PPS", "--mqtt_rate_limit_pps", - default=os.environ.get("WM_SERVICES_MQTT_RATE_LIMIT_PPS", 0), + default=0, action="store", type=self.str2int, help=( @@ -387,9 +448,11 @@ def add_mqtt(self): def add_buffering_settings(self): """ Parameters used to avoid black hole case """ - self.buffering.add_argument( + self.add_env_argument( + self.buffering, + "WM_GW_BUFFERING_MAX_BUFFERED_PACKETS", "--buffering_max_buffered_packets", - default=os.environ.get("WM_GW_BUFFERING_MAX_BUFFERED_PACKETS", 0), + default=0, action="store", type=self.str2int, help=( @@ -398,9 +461,11 @@ def add_buffering_settings(self): ), ) - self.buffering.add_argument( + self.add_env_argument( + self.buffering, + "WM_GW_BUFFERING_MAX_DELAY_WITHOUT_PUBLISH", "--buffering_max_delay_without_publish", - default=os.environ.get("WM_GW_BUFFERING_MAX_DELAY_WITHOUT_PUBLISH", 0), + default=0, action="store", type=self.str2int, help=( @@ -410,9 +475,11 @@ def add_buffering_settings(self): ), ) - self.buffering.add_argument( + self.add_env_argument( + self.buffering, + "WM_GW_BUFFERING_ACTION", "--buffering_action", - default=os.environ.get("WM_GW_BUFFERING_ACTION", None), + default=None, type=BufferingAction, choices=list(BufferingAction), help=( @@ -426,9 +493,11 @@ def add_buffering_settings(self): # This minimal sink cost could be moved somewhere as it can be used even # buffering limitation is not in use - self.buffering.add_argument( + self.add_env_argument( + self.buffering, + "WM_GW_BUFFERING_MINIMAL_SINK_COST", "--buffering_minimal_sink_cost", - default=os.environ.get("WM_GW_BUFFERING_MINIMAL_SINK_COST", 0), + default=0, action="store", type=self.str2int, help=( @@ -439,9 +508,11 @@ def add_buffering_settings(self): ) def add_debug_settings(self): - self.debug.add_argument( + self.add_env_argument( + self.debug, + "WM_SERVICES_DEBUG_INCR_EVENT_ID", "--debug_incr_data_event_id", - default=os.environ.get("WM_SERVICES_DEBUG_INCR_EVENT_ID", False), + default=False, type=self.str2bool, nargs="?", const=True, @@ -523,17 +594,21 @@ def add_deprecated_args(self): help=ParserHelper._deprecated_message("gateway_id"), ) - self.deprecated.add_argument( + self.add_env_argument( + self.deprecated, + "WM_GW_BUFFERING_STOP_STACK", "--buffering_stop_stack", - default=os.environ.get("WM_GW_BUFFERING_STOP_STACK", None), + default=None, type=self.str2bool, - help=ParserHelper._deprecated_message("buffering_stop_stack"), + help=ParserHelper._deprecated_message("buffering_action"), ) def add_gateway_config(self): - self.gateway.add_argument( + self.add_env_argument( + self.gateway, + "WM_GW_ID", "--gateway_id", - default=os.environ.get("WM_GW_ID", None), + default=None, type=self.str2none, help=("Id of the gateway. It must be unique on same broker."), ) @@ -548,45 +623,55 @@ def add_gateway_config(self): help=("Do not use C extension for optimization."), ) - self.gateway.add_argument( + self.add_env_argument( + self.gateway, + "WM_GW_MODEL", "-gm", "--gateway_model", type=self.str2none, - default=os.environ.get("WM_GW_MODEL", None), + default=None, help=("Model name of the gateway."), ) - self.gateway.add_argument( + self.add_env_argument( + self.gateway, + "WM_GW_VERSION", "-gv", "--gateway_version", type=self.str2none, - default=os.environ.get("WM_GW_VERSION", None), + default=None, help=("Version of the gateway."), ) - self.gateway.add_argument( + self.add_env_argument( + self.gateway, + "WM_GW_MAX_SCRAT_SIZE", "-gmss", "--gateway_max_scratchpad_size", type=self.str2int, - default=os.environ.get("WM_GW_MAX_SCRAT_SIZE", None), + default=None, help=("Maximum scratchpad size a gateway can accept. If scratchpad is bigger" "it must be sent as chunks smaller or equal to this value"), ) def add_filtering_config(self): - self.filtering.add_argument( + self.add_env_argument( + self.filtering, + "WM_GW_IGNORED_ENDPOINTS_FILTER", "-iepf", "--ignored_endpoints_filter", type=self.str2none, - default=os.environ.get("WM_GW_IGNORED_ENDPOINTS_FILTER", None), + default=None, help=("Destination endpoints list to ignore (not published)."), ) - self.filtering.add_argument( + self.add_env_argument( + self.filtering, + "WM_GW_WHITENED_ENDPOINTS_FILTER", "-wepf", "--whitened_endpoints_filter", type=self.str2none, - default=os.environ.get("WM_GW_WHITENED_ENDPOINTS_FILTER", None), + default=None, help=( "Destination endpoints list to whiten " "(no payload content, only size)." From 2c1d776f851bcfd344f1492bf6d352a113c41a2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:12:54 +0300 Subject: [PATCH 02/11] Add documentation for buffering and black hole prevention --- .../wirepas_gateway/utils/argument_tools.py | 81 +++++++++++++++++-- 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/python_transport/wirepas_gateway/utils/argument_tools.py b/python_transport/wirepas_gateway/utils/argument_tools.py index 24975d70..72d019e5 100644 --- a/python_transport/wirepas_gateway/utils/argument_tools.py +++ b/python_transport/wirepas_gateway/utils/argument_tools.py @@ -13,6 +13,8 @@ import sys import os import yaml +import textwrap +import shutil from enum import Enum from .serialization_tools import serialize @@ -220,6 +222,22 @@ def add_env_argument(self, kwargs["metavar"] = "$" + env_variable group.add_argument(*args, **kwargs) + def add_wrapped_description(self, target, description, indentation = 2): + """ + Wraps the given description to fit the terminal while keeping line + beaks and adds it to the given target (for example argument group). + """ + width = shutil.get_terminal_size().columns - indentation + lines = [] + for paragraph in description.splitlines(): + if not paragraph: + lines.append("") + continue + wrapped = textwrap.wrap(paragraph, width, replace_whitespace=False) + lines.extend(wrapped) + + target.description = "\n".join(lines) + def add_file_settings(self): """ For file setting handling""" self.add_env_argument( @@ -448,6 +466,57 @@ def add_mqtt(self): def add_buffering_settings(self): """ Parameters used to avoid black hole case """ + self.add_wrapped_description( + self.buffering, + textwrap.dedent("""\ + If the MQTT connection is lost, transport service might end up + buffering uplink packets and never send them to the MQTT broker, + becoming a "black hole". + + Transport service can be configured to detect this and avoid it in + different ways, which can be selected by WM_GW_BUFFERING_ACTION + parameter. By default, transport service will buffer outgoing MQTT + messages without any limit and retry connecting to the broker. + + The black hole prevention can be enabled by setting + WM_GW_BUFFERING_MAX_BUFFERED_PACKETS or + WM_GW_BUFFERING_MAX_DELAY_WITHOUT_PUBLISH parameter. + + Different actions are described below: + + * 'raise_sink_cost' + Sink costs of sinks connected to this gateway are raised to + discourage nodes from connecting to sinks under this gateway. + + * 'stop_stack' + Sinks connected to this gateway are stopped to ensure nodes are + not connected to sinks under this gateway. + + * 'drop_packets' + Enabled only by WM_GW_BUFFERING_MAX_BUFFERED_PACKETS; + WM_GW_BUFFERING_MAX_DELAY_WITHOUT_PUBLISH cannot be used with + this action. Internal publish queue size is limited to + WM_GW_BUFFERING_MAX_BUFFERED_PACKETS and the oldest MQTT messages + are dropped if necessary. Drops are reported in the log + periodically. + + Once the MQTT connection is reestablished, the transport service + waits until all buffered messages have been successfully published + before lowering sink costs or starting sinks again. This is done to + prevent modifying sink parameters in unstable connections with + intermittent disconnects. Also see the WM_SERVICES_MQTT_RATE_LIMIT_PPS + parameter. + + When lowering sink costs, the value of + WM_GW_BUFFERING_MINIMAL_SINK_COST is used. It is also applied to + sinks at startup, even when black hole prevention is disabled: + unlike most sink configuration parameters, sink cost cannot be set + over the MQTT interface, so a raised cost left behind by an earlier + gateway configuration could not be lowered by the backend + otherwise. + """), + ) + self.add_env_argument( self.buffering, "WM_GW_BUFFERING_MAX_BUFFERED_PACKETS", @@ -457,7 +526,7 @@ def add_buffering_settings(self): type=self.str2int, help=( "Maximum number of messages to buffer before " - "taking an action (see --buffering_action). 0 will disable feature" + "taking an action. 0 will disable feature" ), ) @@ -471,7 +540,7 @@ def add_buffering_settings(self): help=( "Maximum time to wait in seconds without any " "successful publish with packet queued " - "before taking an action (see --buffering_action). 0 will disable feature" + "before taking an action. 0 will disable feature" ), ) @@ -482,12 +551,8 @@ def add_buffering_settings(self): default=None, type=BufferingAction, choices=list(BufferingAction), - help=( - "Action to take when the buffer limit is reached. " - "'raise_sink_cost': Increases the sink cost. " - "'stop_stack': Stops the sink stack. " - "'drop_packets': Limits the publish queue size to " - "buffering_max_buffered_packets and drops the oldest packets if necessary." + help=("Action to take when the buffer limit is reached. " + "When empty, it is assumed to be 'raise_sink_cost'." ), ) From daad9426a506f9ae57e7858a9c201481dc7a680f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:50:47 +0300 Subject: [PATCH 03/11] Minor help text fixes to mqtt argument group --- .../wirepas_gateway/utils/argument_tools.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/python_transport/wirepas_gateway/utils/argument_tools.py b/python_transport/wirepas_gateway/utils/argument_tools.py index 72d019e5..54078819 100644 --- a/python_transport/wirepas_gateway/utils/argument_tools.py +++ b/python_transport/wirepas_gateway/utils/argument_tools.py @@ -324,11 +324,7 @@ def add_mqtt(self): default=None, action="store", type=self.str2none, - help=( - "Path to the PEM " - "encoded client private keys " - "respectively." - ), + help=("Path to the PEM encoded client private key."), ) self.add_env_argument( @@ -400,7 +396,7 @@ def add_mqtt(self): type=self.str2bool, nargs="?", const=True, - help=("When True the broker will skip the TLS handshake."), + help=("When true, connect to the broker without TLS."), ) self.add_env_argument( @@ -422,8 +418,9 @@ def add_mqtt(self): action="store", type=self.str2int, help=( - "Delay in seconds to try to reconnect when connection to" - "broker is lost (0 to try forever)" + "Time in seconds to keep trying to reconnect when the " + "connection to the broker is lost. If it expires, the " + "service exits. 0 to retry forever." ), ) @@ -460,7 +457,8 @@ def add_mqtt(self): help=( "Max rate limit for the mqtt client to publish on mqtt broker. It can be set to " "protect the broker from very high usage when one or more gateways are offline for a while " - "and publish all their buffers when connection to broker is restored" + "and publish all their buffers when connection to broker is restored. " + "0 to disable the limit." ), ) From d87f627acf552ef03ace21438d11837720c454d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:03:05 +0300 Subject: [PATCH 04/11] Deprecate WM_SERVICES_MQTT_ALLOW_UNTRUSTED It has never been used earlier; no need to wire it up now --- python_transport/tests/test_arguments.py | 11 ------- .../wirepas_gateway/transport_service.py | 3 ++ .../wirepas_gateway/utils/argument_tools.py | 30 ++++++++++--------- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/python_transport/tests/test_arguments.py b/python_transport/tests/test_arguments.py index 9d2ebc52..31ff5e4d 100644 --- a/python_transport/tests/test_arguments.py +++ b/python_transport/tests/test_arguments.py @@ -17,7 +17,6 @@ # FALSE, means that we don't set it env_vars["WM_SERVICES_MQTT_PERSIST_SESSION"] = True env_vars["WM_SERVICES_MQTT_FORCE_UNSECURE"] = True -env_vars["WM_SERVICES_MQTT_ALLOW_UNTRUSTED"] = True env_vars["WM_GW_BUFFERING_MAX_BUFFERED_PACKETS"] = 1000 env_vars["WM_GW_BUFFERING_MAX_DELAY_WITHOUT_PUBLISH"] = 128 @@ -45,7 +44,6 @@ file_vars["mqtt_ciphers"] = env_vars["WM_SERVICES_MQTT_CIPHERS"] file_vars["mqtt_persist_session"] = env_vars["WM_SERVICES_MQTT_PERSIST_SESSION"] file_vars["mqtt_force_unsecure"] = env_vars["WM_SERVICES_MQTT_FORCE_UNSECURE"] -file_vars["mqtt_allow_untrusted"] = env_vars["WM_SERVICES_MQTT_ALLOW_UNTRUSTED"] file_vars["mqtt_reconnect_delay"] = env_vars["WM_SERVICES_MQTT_RECONNECT_DELAY"] file_vars["buffering_max_buffered_packets"] = env_vars[ "WM_GW_BUFFERING_MAX_BUFFERED_PACKETS" @@ -64,7 +62,6 @@ booleans = [ "WM_SERVICES_MQTT_PERSIST_SESSION", "WM_SERVICES_MQTT_FORCE_UNSECURE", - "WM_SERVICES_MQTT_ALLOW_UNTRUSTED", ] @@ -132,13 +129,6 @@ def content_tests(settings, vcopy): else: assert vcopy["WM_SERVICES_MQTT_FORCE_UNSECURE"] == settings.mqtt_force_unsecure - if "WM_SERVICES_MQTT_ALLOW_UNTRUSTED" not in vcopy: - assert settings.mqtt_allow_untrusted is False - else: - assert ( - vcopy["WM_SERVICES_MQTT_ALLOW_UNTRUSTED"] == settings.mqtt_allow_untrusted - ) - assert vcopy["WM_SERVICES_MQTT_RECONNECT_DELAY"] == settings.mqtt_reconnect_delay assert ( vcopy["WM_GW_BUFFERING_MAX_BUFFERED_PACKETS"] @@ -198,7 +188,6 @@ def test_defaults(): assert settings.mqtt_ciphers is None assert settings.mqtt_persist_session is False assert settings.mqtt_force_unsecure is False - assert settings.mqtt_allow_untrusted is False assert settings.mqtt_reconnect_delay == 0 assert settings.buffering_max_buffered_packets == 0 assert settings.buffering_max_delay_without_publish == 0 diff --git a/python_transport/wirepas_gateway/transport_service.py b/python_transport/wirepas_gateway/transport_service.py index acbf144e..95bac6ee 100644 --- a/python_transport/wirepas_gateway/transport_service.py +++ b/python_transport/wirepas_gateway/transport_service.py @@ -1245,6 +1245,9 @@ def _update_parameters(settings): logging.error("Wrong format for whitened_endpoints_filter EP list (%s)", e) exit() + if settings.mqtt_allow_untrusted: + logging.warning("Param mqtt_allow_untrusted is deprecated and is not in use.") + if settings.buffering_stop_stack is not None: logging.warning("Param buffering_stop_stack is deprecated, please use buffering_action instead") if settings.buffering_action is not None: diff --git a/python_transport/wirepas_gateway/utils/argument_tools.py b/python_transport/wirepas_gateway/utils/argument_tools.py index 54078819..5fa9aed5 100644 --- a/python_transport/wirepas_gateway/utils/argument_tools.py +++ b/python_transport/wirepas_gateway/utils/argument_tools.py @@ -399,17 +399,6 @@ def add_mqtt(self): help=("When true, connect to the broker without TLS."), ) - self.add_env_argument( - self.mqtt, - "WM_SERVICES_MQTT_ALLOW_UNTRUSTED", - "--mqtt_allow_untrusted", - default=False, - type=self.str2bool, - nargs="?", - const=True, - help=("When true the client will skip the certificate name check."), - ) - self.add_env_argument( self.mqtt, "WM_SERVICES_MQTT_RECONNECT_DELAY", @@ -587,14 +576,16 @@ def add_debug_settings(self): ) @staticmethod - def _deprecated_message(new_arg_name, deprecated_from="2.x"): + def _deprecated_message(new_arg_name="", deprecated_from="2.x"): """ Alerts the user that an argument will be deprecated within the next release version """ msg = ( "Deprecated argument (it will be dropped " - "from version {} onwards) please use --{} instead." - ).format(deprecated_from, new_arg_name) + f"from version {deprecated_from} onwards)" + ) + if new_arg_name: + msg += f" please use --{new_arg_name} instead." return msg def add_deprecated_args(self): @@ -657,6 +648,17 @@ def add_deprecated_args(self): help=ParserHelper._deprecated_message("gateway_id"), ) + self.add_env_argument( + self.deprecated, + "WM_SERVICES_MQTT_ALLOW_UNTRUSTED", + "--mqtt_allow_untrusted", + default=False, + type=self.str2bool, + nargs="?", + const=True, + help="Not in use. " + ParserHelper._deprecated_message(), + ) + self.add_env_argument( self.deprecated, "WM_GW_BUFFERING_STOP_STACK", From f39a6feb3ed3825108fae52d0e511c6430f1667c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:22:55 +0300 Subject: [PATCH 05/11] Update help text for gateway argument group --- .../wirepas_gateway/utils/argument_tools.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python_transport/wirepas_gateway/utils/argument_tools.py b/python_transport/wirepas_gateway/utils/argument_tools.py index 5fa9aed5..7637a818 100644 --- a/python_transport/wirepas_gateway/utils/argument_tools.py +++ b/python_transport/wirepas_gateway/utils/argument_tools.py @@ -675,7 +675,13 @@ def add_gateway_config(self): "--gateway_id", default=None, type=self.str2none, - help=("Id of the gateway. It must be unique on same broker."), + help=( + "Id of the gateway. It must be unique on same broker. " + "When empty, an id is generated based on the network " + "interface MAC address (uuid.getnode()). The id is used " + "in MQTT topics without escaping, so special MQTT " + "characters (+, #, /) should be avoided." + ), ) self.gateway.add_argument( @@ -715,7 +721,7 @@ def add_gateway_config(self): "--gateway_max_scratchpad_size", type=self.str2int, default=None, - help=("Maximum scratchpad size a gateway can accept. If scratchpad is bigger" + help=("Maximum scratchpad size a gateway can accept. If scratchpad is bigger " "it must be sent as chunks smaller or equal to this value"), ) From 91b005d0b6e0adc48c3e41dcc1f032e5b013cced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:37:24 +0300 Subject: [PATCH 06/11] Improve documentation for filtering parameters --- .../wirepas_gateway/utils/argument_tools.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/python_transport/wirepas_gateway/utils/argument_tools.py b/python_transport/wirepas_gateway/utils/argument_tools.py index 7637a818..2c1bcfce 100644 --- a/python_transport/wirepas_gateway/utils/argument_tools.py +++ b/python_transport/wirepas_gateway/utils/argument_tools.py @@ -726,6 +726,21 @@ def add_gateway_config(self): ) def add_filtering_config(self): + self.add_wrapped_description( + self.filtering, + textwrap.dedent("""\ + Filters to limit which packets received from the Wirepas + network are published to the MQTT broker. Both filters apply + to uplink traffic only and select packets based on their + destination endpoint. Downlink traffic is never filtered. + + Both parameters accept a list of endpoints (i.e. [1,2,3]), a + range of endpoints (i.e. [1-3]), or a combination of both + (i.e. [1,2,10-15]). Valid endpoint values are 0-255. An + endpoint cannot be in both lists at the same time. + """), + ) + self.add_env_argument( self.filtering, "WM_GW_IGNORED_ENDPOINTS_FILTER", @@ -733,7 +748,10 @@ def add_filtering_config(self): "--ignored_endpoints_filter", type=self.str2none, default=None, - help=("Destination endpoints list to ignore (not published)."), + help=( + "Destination endpoints list to ignore. Packets sent to " + "these endpoints are not published at all." + ), ) self.add_env_argument( @@ -744,8 +762,9 @@ def add_filtering_config(self): type=self.str2none, default=None, help=( - "Destination endpoints list to whiten " - "(no payload content, only size)." + "Destination endpoints list to whiten (i.e. blank out the " + "payload). Packets sent to these endpoints are published " + "without the payload content, only the payload size is kept." ), ) From a77d8c9ee02a335632ca7e144fa79918dd1d5d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:38:26 +0300 Subject: [PATCH 07/11] Fix function documentation for str2none --- python_transport/wirepas_gateway/utils/argument_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_transport/wirepas_gateway/utils/argument_tools.py b/python_transport/wirepas_gateway/utils/argument_tools.py index 2c1bcfce..64911f54 100644 --- a/python_transport/wirepas_gateway/utils/argument_tools.py +++ b/python_transport/wirepas_gateway/utils/argument_tools.py @@ -192,7 +192,7 @@ def str2int(value): @staticmethod def str2none(value): - """ Ensures string to bool conversion """ + """ Converts empty strings to None """ if value == "": return None return value From 15940e1d8f5151f9d7c7c4cc911564b4099f5639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:00:56 +0300 Subject: [PATCH 08/11] Add short description about environment variables in --help --- python_transport/wirepas_gateway/transport_service.py | 9 ++++++++- python_transport/wirepas_gateway/utils/argument_tools.py | 8 ++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/python_transport/wirepas_gateway/transport_service.py b/python_transport/wirepas_gateway/transport_service.py index 95bac6ee..236dba1d 100644 --- a/python_transport/wirepas_gateway/transport_service.py +++ b/python_transport/wirepas_gateway/transport_service.py @@ -10,6 +10,7 @@ from uuid import getnode from threading import Thread, Event from copy import deepcopy +import textwrap from wirepas_gateway.dbus.dbus_client import BusClient from wirepas_gateway.protocol.topic_helper import TopicGenerator, TopicParser @@ -1293,8 +1294,14 @@ def main(): """ parse = ParserHelper( - description="Wirepas Gateway Transport service arguments", version=transport_version, + description=textwrap.dedent("""\ + Wirepas Gateway Transport Service + + Each parameter below can also be set with the environment variable + shown next to it (i.e. $WM_GW_ID). A parameter given on the command + line overrides the environment variable. + """) ) parse.add_file_settings() diff --git a/python_transport/wirepas_gateway/utils/argument_tools.py b/python_transport/wirepas_gateway/utils/argument_tools.py index 64911f54..1ba56c70 100644 --- a/python_transport/wirepas_gateway/utils/argument_tools.py +++ b/python_transport/wirepas_gateway/utils/argument_tools.py @@ -71,14 +71,14 @@ class ParserHelper: def __init__( self, - description="argument parser", + description=None, formatter_class=argparse.RawDescriptionHelpFormatter, version=None, ): super(ParserHelper, self).__init__() - self._parser = argparse.ArgumentParser( - description=description, formatter_class=formatter_class - ) + self._parser = argparse.ArgumentParser(formatter_class=formatter_class) + if description is not None: + self.add_wrapped_description(self._parser, description) self._groups = dict() self._unknown_arguments = None From ce2cf55153891cb8f2e7e9509a2f1d9ba5988910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:32:26 +0300 Subject: [PATCH 09/11] Add cli argument for WM_DEBUG_LEVEL to show it in --help --- .../wirepas_gateway/transport_service.py | 8 +------- .../wirepas_gateway/utils/argument_tools.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/python_transport/wirepas_gateway/transport_service.py b/python_transport/wirepas_gateway/transport_service.py index 236dba1d..bd240e36 100644 --- a/python_transport/wirepas_gateway/transport_service.py +++ b/python_transport/wirepas_gateway/transport_service.py @@ -1314,8 +1314,7 @@ def main(): settings = parse.settings() - # Set default debug level - debug_level = "info" + debug_level = settings.log_level try: debug_level = os.environ["DEBUG_LEVEL"] print( @@ -1326,11 +1325,6 @@ def main(): except KeyError: pass - try: - debug_level = os.environ["WM_DEBUG_LEVEL"] - except KeyError: - pass - debug_level = "{0}".format(debug_level.upper()) # enable its logger diff --git a/python_transport/wirepas_gateway/utils/argument_tools.py b/python_transport/wirepas_gateway/utils/argument_tools.py index 1ba56c70..3a8e9bb4 100644 --- a/python_transport/wirepas_gateway/utils/argument_tools.py +++ b/python_transport/wirepas_gateway/utils/argument_tools.py @@ -560,6 +560,20 @@ def add_buffering_settings(self): ) def add_debug_settings(self): + self.add_env_argument( + self.debug, + "WM_DEBUG_LEVEL", + "--log_level", + default="info", + type=str, + choices=["debug", "info", "warning", "error", "critical"], + help=( + "Log level of the transport service. 'debug' level might " + "generate too much logs and is not recommended to be used in a " + "production system." + ), + ) + self.add_env_argument( self.debug, "WM_SERVICES_DEBUG_INCR_EVENT_ID", From 8dc41a5cdee77ec90a5df4da77222cae44f368c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:58:32 +0300 Subject: [PATCH 10/11] import dbusCExtension in the constructor This allows running the docker container with wm-gw --help to get the help text. Earlier, it was failing because a connection to the system bus was required during import. It might be possible to use lazy import with python 3.15 in the future. --- python_transport/wirepas_gateway/dbus/dbus_client.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python_transport/wirepas_gateway/dbus/dbus_client.py b/python_transport/wirepas_gateway/dbus/dbus_client.py index c8daa89e..06b8d05d 100644 --- a/python_transport/wirepas_gateway/dbus/dbus_client.py +++ b/python_transport/wirepas_gateway/dbus/dbus_client.py @@ -5,7 +5,6 @@ import logging from threading import Thread from pydbus import SystemBus -import dbusCExtension from gi.repository import GLib, GObject from .sink_manager import SinkManager @@ -24,8 +23,12 @@ def __init__(self, cb): """ Thread.__init__(self) - - dbusCExtension.setCallback(cb) + # Imported here instead of module level because importing the C + # extension requires a running system bus. This allows running "--help" + # without connecting to a dbus daemon. + import dbusCExtension + self._dbus_c_extension = dbusCExtension + self._dbus_c_extension.setCallback(cb) self.daemon = True # Daemonize thread def run(self) -> None: @@ -34,7 +37,7 @@ def run(self) -> None: :return: None, as it is an infinite loop in C """ while True: - dbusCExtension.infiniteEventLoop() + self._dbus_c_extension.infiniteEventLoop() logging.error("C extension loop has exited") From 53333d459fa16ff4dc7077b49f666780cc7f070e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Ean=20G=C3=BCne=C5=9F?= <180301198+sgunes-wirepas@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:58:13 +0300 Subject: [PATCH 11/11] Fix some long lines in argument_tools.py --- .../wirepas_gateway/utils/argument_tools.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/python_transport/wirepas_gateway/utils/argument_tools.py b/python_transport/wirepas_gateway/utils/argument_tools.py index 3a8e9bb4..168abe32 100644 --- a/python_transport/wirepas_gateway/utils/argument_tools.py +++ b/python_transport/wirepas_gateway/utils/argument_tools.py @@ -222,7 +222,7 @@ def add_env_argument(self, kwargs["metavar"] = "$" + env_variable group.add_argument(*args, **kwargs) - def add_wrapped_description(self, target, description, indentation = 2): + def add_wrapped_description(self, target, description, indentation=2): """ Wraps the given description to fit the terminal while keeping line beaks and adds it to the given target (for example argument group). @@ -444,10 +444,11 @@ def add_mqtt(self): action="store", type=self.str2int, help=( - "Max rate limit for the mqtt client to publish on mqtt broker. It can be set to " - "protect the broker from very high usage when one or more gateways are offline for a while " - "and publish all their buffers when connection to broker is restored. " - "0 to disable the limit." + "Max rate limit for the mqtt client to publish on mqtt broker. " + "It can be set to protect the broker from very high usage " + "when one or more gateways are offline for a while and " + " publish all their buffers when connection to broker is " + "restored. 0 to disable the limit." ), ) @@ -735,8 +736,11 @@ def add_gateway_config(self): "--gateway_max_scratchpad_size", type=self.str2int, default=None, - help=("Maximum scratchpad size a gateway can accept. If scratchpad is bigger " - "it must be sent as chunks smaller or equal to this value"), + help=( + "Maximum scratchpad size a gateway can accept. If scratchpad " + "is bigger it must be sent as chunks smaller or equal to " + "this value" + ), ) def add_filtering_config(self):