From 6d97becdb14ae07ffcab843ce099fdbe52374b78 Mon Sep 17 00:00:00 2001 From: roshii Date: Sat, 15 Nov 2025 17:15:19 +0100 Subject: [PATCH] feat: tor managed services support --- conftest.py | 17 +++++ docs/onion-message-channels.md | 9 ++- docs/tor.md | 34 ++++++++++ src/jmbase/twisted_utils.py | 104 +++++++++++++++++++++--------- src/jmclient/configure.py | 1 + src/jmdaemon/onionmc.py | 23 +++++-- test/jmbase/test_twisted_utils.py | 65 +++++++++++++++++++ test/jmdaemon/test_onionmc.py | 90 ++++++++++++++++++++++++++ 8 files changed, 304 insertions(+), 39 deletions(-) create mode 100644 test/jmbase/test_twisted_utils.py create mode 100644 test/jmdaemon/test_onionmc.py diff --git a/conftest.py b/conftest.py index 1cc56d582..02659bdac 100644 --- a/conftest.py +++ b/conftest.py @@ -3,6 +3,7 @@ import subprocess from shlex import split from time import sleep +from twisted.internet import reactor from typing import Any, Tuple import pytest @@ -172,3 +173,19 @@ def setup_regtest_bitcoind(pytestconfig): local_command(stop_cmd) # note, it is better to clean out ~/.bitcoin/regtest but too # dangerous to automate it here perhaps + + +@pytest.fixture(autouse=True) +def reset_reactor_state(request): + """Reset reactor _startedBefore flag after twisted.trial tests. + + twisted.trial stops the reactor after each test, which marks it as + _startedBefore=True, preventing subsequent tests from running the reactor. + This fixture resets the flag to allow other tests to use the reactor. + """ + + def reset_flag(): + if hasattr(reactor, "_startedBefore") and reactor._startedBefore: + if not reactor.running: + reactor._startedBefore = False + request.addfinalizer(reset_flag) diff --git a/docs/onion-message-channels.md b/docs/onion-message-channels.md index c21483d9f..57930f387 100644 --- a/docs/onion-message-channels.md +++ b/docs/onion-message-channels.md @@ -59,6 +59,7 @@ onion_serving_port = 8080 # but NOT TO BE USED by non-directory nodes (which is you, unless # you know otherwise!), as it will greatly degrade your privacy. # (note the default is no value, don't replace it with ""). +# Use tor-managed: prefix to use Tor-managed hidden services. hidden_service_dir = # # This is a comma separated list (comma can be omitted if only one item). @@ -155,9 +156,11 @@ Add a non-empty `hidden_service_dir` entry to your `[MESSAGING:onion]` with a di The hostname for your onion service will not change and will be stored permanently in that directory. -The point to understand is: Joinmarket's `jmbase.JMHiddenService` will, if configured with a non-empty `hidden_service_dir` -field, actually start an *independent* instance of Tor specifically for serving this, under the current user. -(our Tor interface library `txtorcon` needs read access to the Tor HS dir, so it's troublesome to do this another way). +There are two ways to configure a persistent hidden service: + +1. **txtorcon-managed** (default when `hidden_service_dir` is set to a path): Joinmarket's `jmbase.JMHiddenService` will manage the hidden service via the Tor control port. This requires control port access and will start an *independent* instance of Tor specifically for serving this, under the current user. (our Tor interface library `txtorcon` needs read access to the Tor HS dir, so it's troublesome to do this another way). + +2. **Tor-managed** (when `hidden_service_dir` is prefixed with `tor-managed:`): Tor manages the hidden service via its `torrc` configuration file. JoinMarket reads the hostname from the `hostname` file in the specified directory. This mode does not require Tor control port access. See [Tor configuration documentation](./tor.md) for setup instructions. #### Question: How to configure the `directory-nodes` list in our `joinmarket.cfg` for this directory node bot? diff --git a/docs/tor.md b/docs/tor.md index 01efa805d..d8eef3c7f 100644 --- a/docs/tor.md +++ b/docs/tor.md @@ -83,3 +83,37 @@ sudo service tor start ``` Once this is done, you should be able to start the yieldgenerator successfully. + +#### Tor-managed hidden services + +As an alternative to using the Tor control port, you can configure Tor to manage the hidden service directly via its configuration file (`torrc`). This approach is useful when: + +- You want Tor to fully manage the hidden service lifecycle +- You don't want to grant control port access to JoinMarket +- You're running Tor as a system service and prefer centralized configuration + +To use this mode: + +1. Configure the hidden service in Tor's `torrc` file (typically `/etc/tor/torrc`): + + ```ini + HiddenServiceDir /var/lib/tor/joinmarket_hidden_service + HiddenServicePort 5222 127.0.0.1:8080 + ``` + +2. Set appropriate permissions +3. Restart To +4. Configure JoinMarket to use the Tor-managed service by setting `hidden_service_dir` in your `joinmarket.cfg`: + + ```ini + hidden_service_dir = tor-managed:/var/lib/tor/joinmarket_hidden_service + ``` + + Note the `tor-managed:` prefix, which tells JoinMarket to read the hostname from the `hostname` file in that directory rather than managing the service via the control port. + +##### Important notes + +- The directory path in `hidden_service_dir` must match exactly what's configured in `torrc` +- JoinMarket will read the hostname from the `hostname` file; make sure Tor has created it +- No control port configuration is needed for this mode (though you may still need it for other features) +- The hidden service directory must be readable by the user running JoinMarket (or the `hostname` file at minimum) diff --git a/src/jmbase/twisted_utils.py b/src/jmbase/twisted_utils.py index ef50b26ad..6933d92f3 100644 --- a/src/jmbase/twisted_utils.py +++ b/src/jmbase/twisted_utils.py @@ -1,13 +1,17 @@ +import os -from zope.interface import implementer +import txtorcon +from twisted.internet import defer, reactor +from twisted.internet.endpoints import ( + TCP4ClientEndpoint, + UNIXClientEndpoint, + serverFromString, +) from twisted.internet.error import ReactorNotRunning -from twisted.internet import reactor, defer -from twisted.internet.endpoints import (TCP4ClientEndpoint, - UNIXClientEndpoint, serverFromString) from twisted.web.client import Agent, BrowserLikePolicyForHTTPS -import txtorcon +from txtorcon import TorConfig, TorControlProtocol from txtorcon.web import tor_agent -from txtorcon import TorControlProtocol, TorConfig +from zope.interface import implementer _custom_stop_reactor_is_set = False custom_stop_reactor = None @@ -170,48 +174,86 @@ def __init__(self, proto_factory_or_resource, info_callback, # an ephemeral HS on the global or pre-existing tor. self.hidden_service_dir = hidden_service_dir + self.tor_connection = None + def start_tor(self): """ This function executes the workflow of starting the hidden service and returning its hostname """ - self.info_callback("Attempting to start onion service on port: {} " - "...".format(self.virtual_port)) + self.info_callback( + f"Attempting to start onion service on port: {self.virtual_port} ..." + ) + + # Check if using Tor-managed mode (via torrc, not control port) + if self.hidden_service_dir.startswith("tor-managed:"): + self.start_tor_managed_onion() + return + + # Ephemeral or txtorcon-managed hidden service (via control port) + if str(self.tor_control_host).startswith("unix:"): + control_endpoint = UNIXClientEndpoint(reactor, self.tor_control_host[5:]) + else: + control_endpoint = TCP4ClientEndpoint( + reactor, self.tor_control_host, self.tor_control_port + ) + d = txtorcon.connect(reactor, control_endpoint) + if self.hidden_service_dir == "": - if str(self.tor_control_host).startswith('unix:'): - control_endpoint = UNIXClientEndpoint(reactor, - self.tor_control_host[5:]) - else: - control_endpoint = TCP4ClientEndpoint(reactor, - self.tor_control_host, self.tor_control_port) - d = txtorcon.connect(reactor, control_endpoint) + # Ephemeral hidden service (no persistence) d.addCallback(self.create_onion_ep) d.addErrback(self.setup_failed) - # TODO: add errbacks to the next two calls in - # the chain: d.addCallback(self.onion_listen) d.addCallback(self.print_host) else: - ep = "onion:" + str(self.virtual_port) + ":localPort=" - ep += str(self.serving_port) - # endpoints.TCPHiddenServiceEndpoint creates version 2 by - # default for backwards compat (err, txtorcon needs to update that ...) - ep += ":version=3" - ep += ":hiddenServiceDir="+self.hidden_service_dir - onion_endpoint = serverFromString(reactor, ep) - d = onion_endpoint.listen(self.proto_factory) + # txtorcon-managed filesystem hidden service + d.addCallback(self.create_filesystem_onion_ep) + d.addErrback(self.setup_failed) d.addCallback(self.print_host_filesystem) - def setup_failed(self, arg): # Note that actions based on this failure are deferred to callers: self.error_callback("Setup failed: " + str(arg)) def create_onion_ep(self, t): self.tor_connection = t - portmap_string = config_to_hs_ports(self.virtual_port, - self.serving_host, self.serving_port) + portmap_string = config_to_hs_ports( + self.virtual_port, self.serving_host, self.serving_port + ) return t.create_onion_service( - ports=[portmap_string], private_key=txtorcon.DISCARD) + ports=[portmap_string], private_key=txtorcon.DISCARD + ) + + def create_filesystem_onion_ep(self, t): + """Create a persistent hidden service using txtorcon's filesystem support. + Requires local Tor control port access. + """ + self.tor_connection = t + ep = "onion:" + str(self.virtual_port) + ":localPort=" + ep += str(self.serving_port) + ep += ":version=3" + ep += ":hiddenServiceDir=" + self.hidden_service_dir + onion_endpoint = serverFromString(reactor, ep) + return onion_endpoint.listen(self.proto_factory) + + def start_tor_managed_onion(self) -> None: + """ + For Tor-managed hidden services: read hostname, start listening. + No control port connection needed. + """ + hs_dir = self.hidden_service_dir.removeprefix("tor-managed:") + hostname_file = os.path.join(hs_dir, "hostname") + + if not os.path.exists(hostname_file): + self.error_callback(f"Hostname file {hostname_file} does not exist") + return + + try: + with open(hostname_file, "r") as f: + hostname = f.read().strip() + self.info_callback(f"Using Tor-managed hidden service: {hostname}") + self.onion_hostname_callback(hostname) + except Exception as e: + self.error_callback(f"Failed to read {hostname_file}: {e}") def onion_listen(self, onion): # 'onion' arg is the created EphemeralOnionService object; @@ -240,11 +282,13 @@ def print_host_filesystem(self, port): self.onion_hostname_callback(self.onion.hostname) def shutdown(self): - self.tor_connection.protocol.transport.loseConnection() + if self.tor_connection: + self.tor_connection.protocol.transport.loseConnection() self.info_callback("Hidden service shutdown complete") if self.shutdown_callback: self.shutdown_callback() + class JMHTTPResource(Resource): """ Object acting as HTTP serving resource """ diff --git a/src/jmclient/configure.py b/src/jmclient/configure.py index bb5559049..03927c48c 100644 --- a/src/jmclient/configure.py +++ b/src/jmclient/configure.py @@ -172,6 +172,7 @@ def jm_single() -> AttributeDict: # but NOT TO BE USED by non-directory nodes (which is you, unless # you know otherwise!), as it will greatly degrade your privacy. # (note the default is no value, don't replace it with ""). +# Use tor-managed: prefix to use Tor-managed hidden services. hidden_service_dir = # # This is a comma separated list (comma can be omitted if only one item). diff --git a/src/jmdaemon/onionmc.py b/src/jmdaemon/onionmc.py index ca5110d8e..3accfeea2 100644 --- a/src/jmdaemon/onionmc.py +++ b/src/jmdaemon/onionmc.py @@ -8,7 +8,7 @@ from twisted.internet import reactor, task, protocol from twisted.protocols import basic from twisted.application.internet import ClientService -from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.internet.endpoints import serverFromString, TCP4ClientEndpoint from twisted.internet.address import IPv4Address, IPv6Address from txtorcon.socks import (TorSocksEndpoint, HostUnreachableError, SocksError, GeneralServerFailureError) @@ -697,9 +697,12 @@ def __init__(self, # it'll fire the `setup_error_callback`. self.hs.start_tor() - # This will serve as our unique identifier, indicating - # that we are ready to communicate (in both directions) over Tor. - self.onion_hostname = None + # For tor-managed services, the hostname is set synchronously by start_tor() + # For ephemeral services, we need to wait for the callback + if not self.hidden_service_dir.startswith("tor-managed:"): + # This will serve as our unique identifier, indicating + # that we are ready to communicate (in both directions) over Tor. + self.onion_hostname = None else: # dummy 'hostname' to indicate we can start running immediately: self.onion_hostname = NOT_SERVING_ONION_HOSTNAME @@ -884,7 +887,7 @@ def connect_to_directories(self) -> None: if self.genesis_node: # we are a directory and we have no directory peers; # just start. - self.on_welcome(self) + self._start_listener() return # the remaining code is only executed by non-directories: for p in self.peers: @@ -901,6 +904,13 @@ def connect_to_directories(self) -> None: self.wait_for_directories) self.wait_for_directories_loop.start(2.0) + def _start_listener(self) -> None: + serverstring = f"tcp:{self.onion_serving_port}:interface={self.onion_serving_host}" + onion_endpoint = serverFromString(reactor, serverstring) + d = onion_endpoint.listen(self.proto_factory) + d.addCallback(self.on_welcome) + d.addErrback(lambda f: self.setup_error_callback(f"Listen failed: {f}")) + def handshake_as_client(self, peer: OnionPeer) -> None: assert peer.status() == PEER_STATUS_CONNECTED if self.self_as_peer.directory: @@ -1461,7 +1471,8 @@ def wait_for_directories(self) -> None: # Note that even if the preceding (max) 50 seconds failed to # connect all our configured dps, we will keep trying and they # can still be used. - if not self.on_welcome_sent: + # For genesis nodes, on_welcome is called after the listener starts + if not self.on_welcome_sent and not self.genesis_node: self.on_welcome(self) self.on_welcome_sent = True self.wait_for_directories_loop.stop() diff --git a/test/jmbase/test_twisted_utils.py b/test/jmbase/test_twisted_utils.py new file mode 100644 index 000000000..9ab70b5d8 --- /dev/null +++ b/test/jmbase/test_twisted_utils.py @@ -0,0 +1,65 @@ +from unittest.mock import Mock, patch + +import pytest + +from jmbase.twisted_utils import JMHiddenService + + +def mock_hs(hidden_service_dir: str = "") -> JMHiddenService: + return JMHiddenService( + Mock(), + Mock(), + Mock(), + Mock(), + "127.0.0.1", + 9051, + "127.0.0.1", + 8080, + 80, + None, + hidden_service_dir, + ) + + +class TestTorManagedHiddenService: + @pytest.mark.parametrize( + "hidden_service_dir,expect_managed,expect_connect", + [ + ("tor-managed:/path/to/dir", True, False), + ("/normal/path", False, True), + ], + ) + def test_hidden_service_dir_detection( + self, hidden_service_dir, expect_managed, expect_connect + ): + with ( + patch.object(JMHiddenService, "start_tor_managed_onion") as mock_managed, + patch("jmbase.twisted_utils.txtorcon.connect") as mock_connect, + ): + hs = mock_hs(hidden_service_dir) + + hs.start_tor() + + if expect_managed: + mock_managed.assert_called_once() + mock_connect.assert_not_called() + else: + mock_managed.assert_not_called() + mock_connect.assert_called_once() + + def test_ephemeral_service_creation(self): + with patch("jmbase.twisted_utils.txtorcon") as mock_txtorcon: + mock_t = Mock() + mock_t.create_onion_service.return_value = Mock() + + hs = mock_hs() + hs.tor_connection = mock_t + hs.virtual_port = 80 + hs.serving_host = "127.0.0.1" + hs.serving_port = 8080 + + hs.create_onion_ep(mock_t) + + mock_t.create_onion_service.assert_called_once_with( + ports=["80 127.0.0.1:8080"], private_key=mock_txtorcon.DISCARD + ) diff --git a/test/jmdaemon/test_onionmc.py b/test/jmdaemon/test_onionmc.py new file mode 100644 index 000000000..c57c8d74c --- /dev/null +++ b/test/jmdaemon/test_onionmc.py @@ -0,0 +1,90 @@ +from unittest.mock import Mock, patch + +import pytest + +from jmdaemon.onionmc import OnionMessageChannel + + +@pytest.fixture +def configdata(): + return { + "btcnet": "mainnet", + "tor_control_host": "127.0.0.1", + "tor_control_port": 9051, + "onion_serving_host": "127.0.0.1", + "serving": False, + "socks5_host": "127.0.0.1", + "socks5_port": 9050, + "directory_nodes": "", + "passive": False, + } + + +class TestOnionMessageChannelListener: + def test_start_listener_creates_tcp_endpoint(self, configdata): + with ( + patch("jmdaemon.onionmc.reactor") as mock_reactor, + patch("jmdaemon.onionmc.serverFromString") as mock_server_from_string, + ): + mock_deferred = Mock() + mock_endpoint = Mock() + mock_endpoint.listen.return_value = mock_deferred + mock_server_from_string.return_value = mock_endpoint + + mc = OnionMessageChannel(configdata) + mc.onion_serving_host = "127.0.0.1" + mc.onion_serving_port = 8080 + mc.proto_factory = Mock() + mc.on_welcome = Mock() + mc.setup_error_callback = Mock() + + mc._start_listener() + + expected_serverstring = "tcp:8080:interface=127.0.0.1" + mock_server_from_string.assert_called_once_with( + mock_reactor, expected_serverstring + ) + + mock_endpoint.listen.assert_called_once_with(mc.proto_factory) + + mock_deferred.addCallback.assert_called_once_with(mc.on_welcome) + mock_deferred.addErrback.assert_called_once() + + errback_callback = mock_deferred.addErrback.call_args[0][0] + test_failure = Exception("Test error") + errback_callback(test_failure) + + mc.setup_error_callback.assert_called_once_with("Listen failed: Test error") + + @pytest.mark.parametrize( + "host,port", + [ + ("192.168.1.1", 9000), + ("localhost", 1234), + ("0.0.0.0", 80), + ], + ) + def test_start_listener_different_ports_and_hosts(self, configdata, host, port): + with ( + patch("jmdaemon.onionmc.reactor") as mock_reactor, + patch("jmdaemon.onionmc.serverFromString") as mock_server_from_string, + ): + mock_endpoint = Mock() + mock_server_from_string.return_value = mock_endpoint + + configdata["onion_serving_host"] = host + configdata["onion_serving_port"] = port + + mc = OnionMessageChannel(configdata) + mc.onion_serving_host = host + mc.onion_serving_port = port + mc.proto_factory = Mock() + mc.on_welcome = Mock() + mc.setup_error_callback = Mock() + + mc._start_listener() + + expected_serverstring = f"tcp:{port}:interface={host}" + mock_server_from_string.assert_called_once_with( + mock_reactor, expected_serverstring + )