diff --git a/.gitattributes b/.gitattributes index 5cb404146d0ff2..2f8f3ece327e1d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,4 @@ -* text=auto +* text=auto eol=lf # to move existing files into LFS: # git add --renormalize . diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b8b1ace97aa27d..ceb66499801971 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -4,6 +4,7 @@ on: push: branches: - master + - windows-dev # --- FORK ONLY, NOT FOR UPSTREAM: CI on pushes to the fork's development branch --- pull_request: workflow_dispatch: workflow_call: @@ -76,6 +77,30 @@ jobs: - name: Building openpilot run: scons + build_windows: + name: build Windows + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + steps: + - uses: actions/checkout@v7 + - uses: msys2/setup-msys2@v2 + with: + msystem: CLANG64 + path-type: inherit # Git for Windows and uv stay visible + - run: ./tools/op.sh setup + - name: Building openpilot + run: tools/op.sh build # activates the venv first: the MSYS2 toolchain's own python3 shadows it in this shell + - name: Software OpenGL for the UI tests + # the runner's display driver has no OpenGL; Mesa's llvmpipe opengl32.dll next to the interpreter wins the DLL search + run: | + pacman -S --needed --noconfirm "$MINGW_PACKAGE_PREFIX-mesa" + source .venv/Scripts/activate + cp /clang64/bin/opengl32.dll "$(dirname "$(python -c 'import sys; print(sys._base_executable)')")" + - name: Run unit tests + timeout-minutes: 20 + run: tools/op.sh test static_analysis: name: static analysis runs-on: ${{ diff --git a/.gitignore b/.gitignore index 54f9f176b73b94..f15a1a3367e854 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,9 @@ bin/ *.os-* *.so *.a +*.dll +*.pyd +*.exe st[0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z] *.unchunked *.clb diff --git a/.gitmodules b/.gitmodules index ad6530de9ac910..ab350cbea65dee 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,15 +1,19 @@ [submodule "panda"] path = panda - url = ../../commaai/panda.git + # --- TODO REMOVE AFTER THE SUBMODULE PRS MERGE (commaai/panda#2427): the pinned commit is on the fork branch behind that PR --- + url = https://github.com/AmyJeanes/panda.git [submodule "opendbc"] path = opendbc_repo - url = ../../commaai/opendbc.git + # --- TODO REMOVE AFTER THE SUBMODULE PRS MERGE (commaai/opendbc#3724): the pinned commit is on the fork branch behind that PR --- + url = https://github.com/AmyJeanes/opendbc.git [submodule "msgq"] path = msgq_repo - url = ../../commaai/msgq.git + # --- TODO REMOVE AFTER THE SUBMODULE PRS MERGE (commaai/msgq#709): the pinned commit is on the fork branch behind that PR --- + url = https://github.com/AmyJeanes/msgq.git [submodule "rednose_repo"] path = rednose_repo - url = ../../commaai/rednose.git + # --- TODO REMOVE AFTER THE SUBMODULE PRS MERGE (commaai/rednose#61): the pinned commit is on the fork branch behind that PR --- + url = https://github.com/AmyJeanes/rednose.git [submodule "teleoprtc_repo"] path = teleoprtc_repo url = ../../commaai/teleoprtc diff --git a/SConstruct b/SConstruct index c6d758b318c08f..292fd42757eb3a 100644 --- a/SConstruct +++ b/SConstruct @@ -4,6 +4,8 @@ import sys import sysconfig import platform import shlex +import shutil +import tempfile import importlib import numpy as np @@ -43,16 +45,21 @@ if external_pythonpath := os.environ.get("PYTHONPATH"): submodule_python_paths += [p for p in external_pythonpath.split(os.pathsep) if p and p not in submodule_python_paths] # Detect platform -arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() +WINDOWS = platform.system() == "Windows" if platform.system() == "Darwin": arch = "Darwin" -elif arch == "aarch64" and COMMA_HARDWARE: - arch = "comma_arm64" +elif WINDOWS: + arch = "Windows" +else: + arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() + if arch == "aarch64" and COMMA_HARDWARE: + arch = "comma_arm64" assert arch in [ "comma_arm64", # linux comma hardware (AGNOS) arm64 "aarch64", # linux pc arm64 "x86_64", # linux pc x64 "Darwin", # macOS arm64 (x86 not supported) + "Windows", # windows pc x64, development only (MSYS2 clang64 toolchain) ] pkg_names = ['acados', 'capnproto', 'ffmpeg', 'json11', 'ncurses', 'zeromq', 'zstd'] @@ -65,13 +72,13 @@ ffmpeg = pkgs[pkg_names.index('ffmpeg')] # TODO: drop the static fallback once device venvs have comma-deps-ffmpeg>=7.1.0.post94 _ffmpeg_lib_names = os.listdir(ffmpeg.LIB_DIR) if os.path.isdir(ffmpeg.LIB_DIR) else [] ffmpeg_shared = any( - n.startswith('libavcodec.so') or (n.startswith('libavcodec') and n.endswith('.dylib')) + n.startswith('libavcodec.so') or (n.startswith('libavcodec') and n.endswith(('.dylib', '.dll.a'))) for n in _ffmpeg_lib_names ) ffmpeg_libs = ['avformat', 'avcodec', 'swresample', 'avutil'] if not ffmpeg_shared: ffmpeg_libs += ['x264', 'z'] - if arch != "Darwin": + if arch not in ("Darwin", "Windows"): ffmpeg_libs += ['va', 'va-drm', 'drm'] acados_include_dirs = [ acados.INCLUDE_DIR, @@ -88,15 +95,22 @@ acados_include_dirs = [ allowed_system_libs = { "EGL", "GLESv2", "GL", "dl", "drm", "gbm", "m", "pthread", + # Windows SDK import libraries + "opengl32", "gdi32", "winmm", "shell32", "user32", "advapi32", "ws2_32", "bcrypt", "ole32", "setupapi", "shlwapi", "dwmapi", "ntdll", + "iphlpapi", "rpcrt4", } +# static libzmq/capnp need these on every Windows link; import libs only pull in what is referenced +windows_link_libs = ["pthread", "ws2_32", "iphlpapi", "rpcrt4", "bcrypt", "advapi32", "ole32", "user32", "shell32"] if WINDOWS else [] def _resolve_lib(env, name): for d in env.Flatten(env.get('LIBPATH', [])): p = Dir(str(d)).abspath - for ext in ('.a', '.so', '.dylib'): + for ext in ('.a', '.so', '.dylib', '.dll.a', '.lib'): f = File(os.path.join(p, f'lib{name}{ext}')) if f.exists() or f.has_builder(): return name + if WINDOWS and File(os.path.join(p, f'{name}.lib')).exists(): # MSVC-style import library, e.g. python312.lib + return name if name in allowed_system_libs: return name raise SCons.Errors.UserError(f"Unexpected non-vendored library '{name}'") @@ -114,11 +128,39 @@ def _libflags(target, source, env, for_signature): libs.append(_resolve_lib(env, lib)) else: libs.append(lib) + libs += [_resolve_lib(env, lib) for lib in windows_link_libs] return _stripixes(env['LIBLINKPREFIX'], libs, env['LIBLINKSUFFIX'], env['LIBPREFIXES'], env['LIBSUFFIXES'], env, env['LIBLITERALPREFIX']) +if WINDOWS: + # Run every build command through MSYS2 bash so the POSIX shell syntax used by the + # SConscripts (cd x && ..., VAR=1 ./script.py, shebang scripts) keeps working. This + # replaces the platform spawn so it also covers Environments created by submodules. + # bash must be resolved via PATH: CreateProcess searches System32 first, which + # would pick the WSL launcher. + import SCons.Platform.posix + import SCons.Platform.win32 + _bash = shutil.which("bash") + if not _bash or "system32" in _bash.lower(): + raise SCons.Errors.UserError("MSYS2 bash must be on PATH before System32 (run scons from an MSYS2 CLANG64 shell)") + + def _bash_spawn(sh, escape, cmd, args, env): + # bash treats backslashes as escapes, so hand it SCons' Windows paths with forward + # slashes. Arguments carrying quotes (defines such as -DSWAGLOG="\"...\"") are kept as is. + args = [a if '"' in a else a.replace("\\", "/") for a in args] + return subprocess.call([_bash, "-c", " ".join(args)], env=env) + SCons.Platform.win32.spawn = _bash_spawn + SCons.Platform.win32.escape = SCons.Platform.posix.escape + +# Windows child processes need the system variables cmd/python rely on; POSIX builds keep the strict env +_windows_env = {k: os.environ[k] for k in ( + "SYSTEMROOT", "SYSTEMDRIVE", "COMSPEC", "PATHEXT", "TEMP", "TMP", "USERPROFILE", "HOMEDRIVE", "HOMEPATH", "HOME", + "APPDATA", "LOCALAPPDATA", "USERNAME", "NUMBER_OF_PROCESSORS", "PROCESSOR_ARCHITECTURE", "MSYSTEM", "MSYSTEM_PREFIX", "VIRTUAL_ENV", +) if k in os.environ} if WINDOWS else {} + env = Environment( ENV={ + **_windows_env, "PATH": os.environ['PATH'], "PYTHONPATH": os.pathsep.join(submodule_python_paths), "ACADOS_SOURCE_DIR": acados.DIR, @@ -132,7 +174,7 @@ env = Environment( "-O2", "-Wunused", "-Werror", - "-Wshadow" if arch in ("Darwin", "comma_arm64") else "-Wshadow=local", + "-Wshadow" if arch in ("Darwin", "comma_arm64", "Windows") else "-Wshadow=local", "-Wno-unknown-warning-option", "-Wno-inconsistent-missing-override", "-Wno-c99-designator", @@ -163,7 +205,7 @@ env = Environment( CYTHONCFILESUFFIX=".cpp", COMPILATIONDB_USE_ABSPATH=True, REDNOSE_ROOT="#rednose_repo", - tools=["default", "cython", "compilation_db", "rednose_filter"], + tools=["mingw" if WINDOWS else "default", "cython", "compilation_db", "rednose_filter"], toolpath=["#msgq_repo/site_scons/site_tools", "#rednose_repo/site_scons/site_tools"], ) # SCons' Darwin linker tool doesn't define the variables used to expand RPATH. @@ -173,6 +215,20 @@ if arch == "Darwin": env["_RPATH"] = "${_concat(RPATHPREFIX, RPATH, RPATHSUFFIX, __env__)}" if arch != "comma_arm64": env['_LIBFLAGS'] = _libflags +if WINDOWS: + # clang and lld through the mingw tool, whose defaults are gcc; shared libraries keep the lib prefix the SConscripts expect + env["CC"], env["CXX"] = "clang", "clang++" + env["SHLIBPREFIX"] = "lib" + # PE has no rpath; DLLs are found next to the executable or via PATH + env["_RPATH"] = "" + # Self-contained binaries: Python 3.8+ does not search PATH for the DLLs an extension + # module needs, so the libc++/winpthreads runtime must not be dynamic + env.Append(LINKFLAGS=["-static"]) + + # uv venvs on Windows only ship python.exe, but scripts and shebangs here expect python3 + _python3 = os.path.join(os.path.dirname(sys.executable), "python3.exe") + if not os.path.exists(_python3): + shutil.copy2(sys.executable, _python3) # Arch-specific flags and paths if arch == "comma_arm64": @@ -190,6 +246,10 @@ elif arch == "Darwin": ]) env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"]) env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"]) +elif arch == "Windows": + # -std=c++1z is strict ANSI; mingw then hides vasprintf, M_PI and friends without these + # ZMQ_STATIC: the vendored libzmq is a static archive, without it zmq.h asks for DLL imports + env.Append(CCFLAGS=["-D_GNU_SOURCE", "-D_USE_MATH_DEFINES", "-DZMQ_STATIC"]) _extra_cc = shlex.split(GetOption('ccflags') or '') if _extra_cc: @@ -217,13 +277,18 @@ if not GetOption('verbose'): # ********** Cython build environment ********** envCython = env.Clone() -envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()] +# in a Windows venv sysconfig points at the (empty) venv Include dir, headers live with the base interpreter +envCython["CPPPATH"] += [sysconfig.get_paths(vars={"installed_base": sys.base_prefix})['include'], np.get_include()] envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"] envCython["CCFLAGS"].remove("-Werror") envCython["LIBS"] = [] if arch == "Darwin": envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"] +elif arch == "Windows": + envCython["LINKFLAGS"] = ["-shared", "-static"] + envCython["LIBPATH"] += [os.path.join(sys.base_prefix, "libs")] + envCython["LIBS"] += [f"python{sys.version_info.major}{sys.version_info.minor}"] else: envCython["LINKFLAGS"] = ["-pthread", "-shared"] @@ -233,7 +298,7 @@ Export('envCython', 'np_version') Export('env', 'arch', 'acados', 'ffmpeg_libs') # Setup cache dir -cache_dir = '/data/scons_cache' if arch == "comma_arm64" else '/tmp/scons_cache' +cache_dir = '/data/scons_cache' if arch == "comma_arm64" else os.path.join(tempfile.gettempdir(), 'scons_cache') if WINDOWS else '/tmp/scons_cache' cache_size_limit = 4e9 if "CI" in os.environ else 2e9 CacheDir(cache_dir) Clean(["."], cache_dir) diff --git a/msgq_repo b/msgq_repo index 0e266c1dbcf732..998acf78a4635f 160000 --- a/msgq_repo +++ b/msgq_repo @@ -1 +1 @@ -Subproject commit 0e266c1dbcf7328beee3e57b4a8688555387c877 +Subproject commit 998acf78a4635f60d49bba2f0d6390efdce5c7ee diff --git a/opendbc_repo b/opendbc_repo index b4ef5e1cf406ff..6c706cf3966931 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit b4ef5e1cf406ff143fa67bdbfb154739d43279c9 +Subproject commit 6c706cf39669318bebbfece7828503b2edeffc72 diff --git a/openpilot/__init__.py b/openpilot/__init__.py index e69de29bb2d1d6..9b5da58d7e9fb2 100644 --- a/openpilot/__init__.py +++ b/openpilot/__init__.py @@ -0,0 +1,8 @@ +import sys +import time + +if sys.platform == "win32" and sys.version_info < (3, 13): + # the GetTickCount64 based clock only ticks every 15.6 ms; Python 3.13 moved monotonic to QueryPerformanceCounter + # TODO: drop when the Python pin reaches 3.13 + setattr(time, "monotonic", time.perf_counter) # noqa: B010 (a plain assignment is a type error for ty on Windows) + setattr(time, "monotonic_ns", time.perf_counter_ns) # noqa: B010 diff --git a/openpilot/cereal/messaging/tests/test_messaging.py b/openpilot/cereal/messaging/tests/test_messaging.py index 462f9dc2a7832a..eccedf1146cec8 100644 --- a/openpilot/cereal/messaging/tests/test_messaging.py +++ b/openpilot/cereal/messaging/tests/test_messaging.py @@ -41,6 +41,11 @@ def assert_carstate(cs1, cs2): if isinstance(val1, numbers.Number): assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'" +def recv_one_retry_process(sock, timeout): + # module level: Windows spawns the process, and a socket cannot be pickled into it + messaging.recv_one_retry(messaging.sub_sock(sock, timeout=round(timeout * 1000))) + + def delayed_send(delay, sock, dat): def send_func(): sock.send(dat) @@ -148,7 +153,7 @@ def test_recv_one_retry(self): sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000)) # wait 5 socket timeouts and make sure it's still retrying - p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,)) + p = multiprocessing.Process(target=recv_one_retry_process, args=(sock, sock_timeout)) p.start() time.sleep(sock_timeout*5) assert p.is_alive() diff --git a/openpilot/cereal/messaging/tests/test_services.py b/openpilot/cereal/messaging/tests/test_services.py index f4c1b81e4f1c6d..ebc306fc876e0b 100644 --- a/openpilot/cereal/messaging/tests/test_services.py +++ b/openpilot/cereal/messaging/tests/test_services.py @@ -17,6 +17,7 @@ def test_services(self, s): assert service.decimation != 0 def test_generated_header(self): - with tempfile.NamedTemporaryFile(suffix=".h") as f: + with tempfile.NamedTemporaryFile(suffix=".h", delete_on_close=False) as f: + f.close() # Windows: other processes cannot open the file while it is open here ret = subprocess.run(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11", shell=True).returncode assert ret == 0, "generated services header is not valid C" diff --git a/openpilot/common/SConscript b/openpilot/common/SConscript index 4e06ae88a155d5..9af86483a5fee2 100644 --- a/openpilot/common/SConscript +++ b/openpilot/common/SConscript @@ -2,6 +2,7 @@ Import('env') common_libs = [ 'params.cc', + 'prefix.cc', 'swaglog.cc', 'util.cc', 'ratekeeper.cc', diff --git a/openpilot/common/file.h b/openpilot/common/file.h new file mode 100644 index 00000000000000..c4747f2de09dd3 --- /dev/null +++ b/openpilot/common/file.h @@ -0,0 +1,14 @@ +#pragma once + +// File primitives whose POSIX spelling differs on Windows. Kept apart from util.h: that header's Rect collides with +// the one MacTypes.h brings in when a translation unit also includes CoreFoundation on macOS (cabana's settings). +#ifdef _WIN32 +#include +inline int fsync(int fd) { return _commit(fd); } +#endif + +namespace util { +// an exclusive lock held until fd closes, and a rename that replaces an existing target (rename() refuses to on Windows) +int lock_file_exclusive(int fd); +int replace_file(const char *from, const char *to); +} // namespace util diff --git a/openpilot/common/gpio.py b/openpilot/common/gpio.py index 8f025a2daf726e..f40259893fa9ec 100644 --- a/openpilot/common/gpio.py +++ b/openpilot/common/gpio.py @@ -1,6 +1,9 @@ import os -import fcntl import ctypes +try: + import fcntl +except ImportError: # Windows: only importable there, the ioctls need comma hardware + fcntl = None from functools import cache def gpio_init(pin: int, output: bool) -> None: diff --git a/openpilot/common/hardware/hw.h b/openpilot/common/hardware/hw.h index 83dc452da85c66..889a18fc666f78 100644 --- a/openpilot/common/hardware/hw.h +++ b/openpilot/common/hardware/hw.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "common/hardware/base.h" @@ -19,7 +20,11 @@ namespace Path { } inline std::string comma_home() { +#ifdef _WIN32 + return util::getenv("USERPROFILE") + "/.comma" + Path::openpilot_prefix(); // what Python's Path.home() uses +#else return util::getenv("HOME") + "/.comma" + Path::openpilot_prefix(); +#endif } inline std::string log_root() { @@ -38,19 +43,39 @@ namespace Path { } inline std::string swaglog_ipc() { +#ifdef _WIN32 + // libzmq has no ipc:// transport on MinGW: derive a loopback port from the prefix (FNV-1a, mirrored in hw.py) + uint64_t h = 14695981039346656037ULL; + for (unsigned char c : Path::openpilot_prefix()) { + h ^= c; + h *= 1099511628211ULL; + } + return "tcp://127.0.0.1:" + std::to_string(26000 + h % 1000); +#else return "ipc:///tmp/logmessage" + Path::openpilot_prefix(); +#endif + } + + inline std::string tmp_dir() { // Python's tempfile.gettempdir() +#ifdef _WIN32 + return util::getenv("TEMP", "."); +#else + return "/tmp"; +#endif } inline std::string download_cache_root() { if (const char *env = getenv("COMMA_CACHE")) { return env; } - return "/tmp/comma_download_cache" + Path::openpilot_prefix() + "/"; + return tmp_dir() + "/comma_download_cache" + Path::openpilot_prefix() + "/"; } inline std::string shm_path() { #ifdef __APPLE__ return"/tmp"; + #elif defined(_WIN32) + return tmp_dir(); #else return "/dev/shm"; #endif diff --git a/openpilot/common/hardware/hw.py b/openpilot/common/hardware/hw.py index 1041a17c1c7fc4..c5cbbef0e08d0e 100644 --- a/openpilot/common/hardware/hw.py +++ b/openpilot/common/hardware/hw.py @@ -1,10 +1,13 @@ import os import platform +import sys +import tempfile from pathlib import Path from openpilot.common.hardware import PC -DEFAULT_DOWNLOAD_CACHE_ROOT = "/tmp/comma_download_cache" +TMP_DIR = tempfile.gettempdir() if sys.platform == "win32" else "/tmp" # Path::tmp_dir() +DEFAULT_DOWNLOAD_CACHE_ROOT = os.path.join(TMP_DIR, "comma_download_cache") class Paths: @staticmethod @@ -29,7 +32,14 @@ def swaglog_root() -> str: @staticmethod def swaglog_ipc() -> str: - return "ipc:///tmp/logmessage" + os.environ.get("OPENPILOT_PREFIX", "") + prefix = os.environ.get("OPENPILOT_PREFIX", "") + if sys.platform == "win32": + # libzmq has no ipc:// transport on MinGW: derive a loopback port from the prefix (FNV-1a, mirrored in hw.h) + h = 14695981039346656037 + for c in prefix.encode(): + h = ((h ^ c) * 1099511628211) & 0xFFFFFFFFFFFFFFFF + return f"tcp://127.0.0.1:{26000 + h % 1000}" + return "ipc:///tmp/logmessage" + prefix @staticmethod def download_cache_root() -> str: @@ -55,4 +65,6 @@ def config_root() -> str: def shm_path() -> str: if PC and platform.system() == "Darwin": return "/tmp" # This is not really shared memory on macOS, but it's the closest we can get + if sys.platform == "win32": + return tempfile.gettempdir() # msgq reads %TEMP% for the same directory return "/dev/shm" diff --git a/openpilot/common/i2c.py b/openpilot/common/i2c.py index 1dfaa659ad302e..3984718cf46d03 100644 --- a/openpilot/common/i2c.py +++ b/openpilot/common/i2c.py @@ -1,6 +1,9 @@ import os -import fcntl import ctypes +try: + import fcntl +except ImportError: # Windows: only importable there, the ioctls need comma hardware + fcntl = None # I2C constants from /usr/include/linux/i2c-dev.h I2C_SLAVE = 0x0703 diff --git a/openpilot/common/params.cc b/openpilot/common/params.cc index 495feb0a99111a..db05dcd9c8e2c4 100644 --- a/openpilot/common/params.cc +++ b/openpilot/common/params.cc @@ -1,11 +1,15 @@ #include "common/params.h" +#ifdef _WIN32 +#include +#else #include -#include +#endif #include #include #include +#include #include #include "common/params_keys.h" @@ -22,6 +26,10 @@ void params_sig_handler(int signal) { } int fsync_dir(const std::string &path) { +#ifdef _WIN32 + (void)path; + return 0; // directories cannot be opened through the CRT; NTFS journals the rename +#endif int result = -1; int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755)); if (fd >= 0) { @@ -39,6 +47,12 @@ bool create_params_path(const std::string ¶m_path, const std::string &key_pa // See if the symlink exists, otherwise create it if (!util::file_exists(key_path)) { +#ifdef _WIN32 + // symlinks need privileges on Windows; a plain directory does for development + if (_mkdir(key_path.c_str()) != 0 && errno != EEXIST) { + return false; + } +#else // 1) Create temp folder // 2) Symlink it to temp link // 3) Move symlink to /d @@ -59,6 +73,7 @@ bool create_params_path(const std::string ¶m_path, const std::string &key_pa if (rename(link_path.c_str(), key_path.c_str()) != 0 && errno != EEXIST) { return false; } +#endif } return true; @@ -78,7 +93,7 @@ class FileLock { public: FileLock(const std::string &fn) { fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775)); - if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) { + if (fd_ < 0 || util::lock_file_exclusive(fd_) < 0) { LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno); } } @@ -149,17 +164,22 @@ int Params::put(const char* key, const char* value, size_t value_size) { // fsync to force persist the changes. if ((result = HANDLE_EINTR(fsync(tmp_fd))) < 0) break; +#ifdef _WIN32 + // an open file cannot be renamed on Windows + close(tmp_fd); + tmp_fd = -1; +#endif FileLock file_lock(params_path + "/.lock"); // Move temp into place. - if ((result = rename(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break; + if ((result = util::replace_file(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break; // fsync parent directory result = fsync_dir(getParamPath()); } while (false); - close(tmp_fd); + if (tmp_fd >= 0) close(tmp_fd); if (result != 0) { ::unlink(tmp_path.c_str()); } @@ -208,6 +228,17 @@ void Params::clearAll(ParamKeyFlag key_flag) { // 1) delete params of key_flag // 2) delete files that are not defined in the keys. +#ifdef _WIN32 + std::error_code ec; + for (const auto &entry : std::filesystem::directory_iterator(getParamPath(), ec)) { + if (entry.is_directory()) continue; + std::string name = entry.path().filename().string(); + auto it = keys.find(name); + if (it == keys.end() || (it->second.flags & key_flag)) { + unlink(getParamPath(name).c_str()); + } + } +#else if (DIR *d = opendir(getParamPath().c_str())) { struct dirent *de = NULL; while ((de = readdir(d))) { @@ -220,6 +251,7 @@ void Params::clearAll(ParamKeyFlag key_flag) { } closedir(d); } +#endif fsync_dir(getParamPath()); } diff --git a/openpilot/common/params.py b/openpilot/common/params.py index 9357a0a5d34966..74b08860909b8d 100644 --- a/openpilot/common/params.py +++ b/openpilot/common/params.py @@ -30,7 +30,7 @@ class ParamKeyType(IntEnum): BYTES = 6 -_suffix = ".dylib" if sys.platform == "darwin" else ".so" +_suffix = {"darwin": ".dylib", "win32": ".dll"}.get(sys.platform, ".so") lib = ctypes.CDLL(Path(__file__).with_name(f"libparams_c{_suffix}")) ParamsHandle = ctypes.c_void_p diff --git a/openpilot/common/prefix.cc b/openpilot/common/prefix.cc new file mode 100644 index 00000000000000..5ea472b4ffd8b5 --- /dev/null +++ b/openpilot/common/prefix.cc @@ -0,0 +1,39 @@ +#include "common/prefix.h" + +#include +#include + +#include "common/params.h" +#include "common/util.h" +#include "common/hardware/hw.h" + +OpenpilotPrefix::OpenpilotPrefix(std::string prefix) { + if (prefix.empty()) { + prefix = util::random_string(15); + } + msgq_path = Path::shm_path() + "/msgq_" + prefix; + bool ret = util::create_directories(msgq_path, 0777); + assert(ret); + setenv("OPENPILOT_PREFIX", prefix.c_str(), 1); +} + +OpenpilotPrefix::~OpenpilotPrefix() { + // best effort: Windows refuses to delete queue files that sockets in this process still map + std::error_code ec; + auto param_path = Params().getParamPath(); + if (util::file_exists(param_path)) { +#ifdef _WIN32 + std::filesystem::remove_all(param_path, ec); // a plain directory, see params.cc +#else + std::string real_path = util::readlink(param_path); + util::check_system(util::string_format("rm -rf %s", real_path.c_str())); + unlink(param_path.c_str()); +#endif + } + if (getenv("COMMA_CACHE") == nullptr) { + std::filesystem::remove_all(Path::download_cache_root(), ec); + } + std::filesystem::remove_all(Path::comma_home(), ec); + std::filesystem::remove_all(msgq_path, ec); + unsetenv("OPENPILOT_PREFIX"); +} diff --git a/openpilot/common/prefix.h b/openpilot/common/prefix.h index 0f2c592527913b..a4376161dcc93f 100644 --- a/openpilot/common/prefix.h +++ b/openpilot/common/prefix.h @@ -1,42 +1,11 @@ #pragma once -#include #include -#include "common/params.h" -#include "common/util.h" -#include "common/hardware/hw.h" - class OpenpilotPrefix { public: - OpenpilotPrefix(std::string prefix = {}) { - if (prefix.empty()) { - prefix = util::random_string(15); - } -#ifdef __APPLE__ - msgq_path = "/tmp/msgq_" + prefix; -#else - msgq_path = "/dev/shm/msgq_" + prefix; -#endif - bool ret = util::create_directories(msgq_path, 0777); - assert(ret); - setenv("OPENPILOT_PREFIX", prefix.c_str(), 1); - } - - ~OpenpilotPrefix() { - auto param_path = Params().getParamPath(); - if (util::file_exists(param_path)) { - std::string real_path = util::readlink(param_path); - util::check_system(util::string_format("rm -rf %s", real_path.c_str())); - unlink(param_path.c_str()); - } - if (getenv("COMMA_CACHE") == nullptr) { - util::check_system(util::string_format("rm -rf %s", Path::download_cache_root().c_str())); - } - util::check_system(util::string_format("rm -rf %s", Path::comma_home().c_str())); - util::check_system(util::string_format("rm -rf %s", msgq_path.c_str())); - unsetenv("OPENPILOT_PREFIX"); - } + OpenpilotPrefix(std::string prefix = {}); + ~OpenpilotPrefix(); private: std::string msgq_path; diff --git a/openpilot/common/prefix.py b/openpilot/common/prefix.py index d0be8997ae1ac6..a710fe462a5e55 100644 --- a/openpilot/common/prefix.py +++ b/openpilot/common/prefix.py @@ -1,5 +1,4 @@ import os -import platform import shutil import uuid @@ -12,8 +11,7 @@ class OpenpilotPrefix: def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False): self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15]) - shm_path = "/tmp" if platform.system() == "Darwin" else "/dev/shm" - self.msgq_path = os.path.join(shm_path, "msgq_" + self.prefix) + self.msgq_path = os.path.join(Paths.shm_path(), "msgq_" + self.prefix) self.create_dirs_on_enter = create_dirs_on_enter self.clean_dirs_on_exit = clean_dirs_on_exit self.shared_download_cache = shared_download_cache @@ -52,7 +50,8 @@ def clean_dirs(self): symlink_path = Params().get_param_path() if os.path.exists(symlink_path): shutil.rmtree(os.path.realpath(symlink_path), ignore_errors=True) - os.remove(symlink_path) + if os.path.islink(symlink_path): # a plain directory on Windows, see params.cc + os.remove(symlink_path) shutil.rmtree(self.msgq_path, ignore_errors=True) if PC: shutil.rmtree(Paths.log_root(), ignore_errors=True) diff --git a/openpilot/common/swaglog.cc b/openpilot/common/swaglog.cc index 74c617fa611008..d7c9f2b68ca6ce 100644 --- a/openpilot/common/swaglog.cc +++ b/openpilot/common/swaglog.cc @@ -62,6 +62,10 @@ class SwaglogState { } ~SwaglogState() { +#ifdef _WIN32 + // runs from DllMain at process exit, after the zmq I/O thread is gone: zmq_ctx_destroy() would wait forever + return; +#endif zmq_close(sock); zmq_ctx_destroy(zctx); } diff --git a/openpilot/common/tests/test_file_helpers.py b/openpilot/common/tests/test_file_helpers.py index 09b6990ed60554..91cdcd4ab3dbfa 100644 --- a/openpilot/common/tests/test_file_helpers.py +++ b/openpilot/common/tests/test_file_helpers.py @@ -1,13 +1,14 @@ import os from uuid import uuid4 +from openpilot.common.hardware.hw import TMP_DIR from openpilot.common.test import OpenpilotTestCase from openpilot.common.utils import atomic_write class TestFileHelpers(OpenpilotTestCase): def run_atomic_write_func(self, atomic_write_func): - path = f"/tmp/tmp{uuid4()}" + path = os.path.join(TMP_DIR, f"tmp{uuid4()}") with atomic_write_func(path) as f: f.write("test") assert not os.path.exists(path) diff --git a/openpilot/common/timeout.py b/openpilot/common/timeout.py index d0b0ce0630af78..9e5e6036f5c520 100644 --- a/openpilot/common/timeout.py +++ b/openpilot/common/timeout.py @@ -1,4 +1,6 @@ import signal +import sys +import threading class TimeoutException(Exception): pass @@ -9,6 +11,9 @@ class Timeout: For example this code will raise a TimeoutException: with Timeout(seconds=5, error_msg="Sleep was too long"): time.sleep(10) + + On Windows the timeout interrupts Python code and sleeps, but not a blocking wait on a + child process or pipe: that only raises once the wait itself returns. """ def __init__(self, seconds, error_msg=None): if error_msg is None: @@ -20,8 +25,24 @@ def handle_timeout(self, signume, frame): raise TimeoutException(self.error_msg) def __enter__(self): + if sys.platform == "win32": + # no SIGALRM: a timer thread raises SIGINT, which becomes a KeyboardInterrupt in the main thread and + # also wakes time.sleep (interrupt_main would not); __exit__ translates it + self.expired = False + self.timer = threading.Timer(self.seconds, self._interrupt) + self.timer.start() + return signal.signal(signal.SIGALRM, self.handle_timeout) signal.alarm(self.seconds) + def _interrupt(self): + self.expired = True + signal.raise_signal(signal.SIGINT) + def __exit__(self, exc_type, exc_val, exc_tb): + if sys.platform == "win32": + self.timer.cancel() + if self.expired and exc_type is KeyboardInterrupt: + raise TimeoutException(self.error_msg) from None + return signal.alarm(0) diff --git a/openpilot/common/timing.h b/openpilot/common/timing.h index 83f55e0c4009f5..c3a353a7a395d8 100644 --- a/openpilot/common/timing.h +++ b/openpilot/common/timing.h @@ -3,9 +3,12 @@ #include #include -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(_WIN32) #define CLOCK_BOOTTIME CLOCK_MONOTONIC #endif +#ifdef _WIN32 +#define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC +#endif static inline uint64_t nanos_since_boot() { struct timespec t; diff --git a/openpilot/common/util.cc b/openpilot/common/util.cc index 84b47e187ee05e..80ae87c9d725d4 100644 --- a/openpilot/common/util.cc +++ b/openpilot/common/util.cc @@ -1,14 +1,23 @@ +#ifdef _WIN32 +#include "common/win32.h" +#include +#endif + #include "common/util.h" #include "common/swaglog.h" -#include #include +#ifndef _WIN32 +#include +#include #include +#endif #include #include #include #include +#include #include #include #include @@ -63,6 +72,10 @@ int set_core_affinity(std::vector cores) { } int set_file_descriptor_limit(uint64_t limit_val) { +#ifdef _WIN32 + (void)limit_val; + return 0; +#else struct rlimit limit; int status; @@ -74,6 +87,7 @@ int set_file_descriptor_limit(uint64_t limit_val) { return status; return 0; +#endif } std::string read_file(const std::string& fn) { @@ -102,6 +116,14 @@ std::string read_file(const std::string& fn) { std::map read_files_in_dir(const std::string &path) { std::map ret; +#ifdef _WIN32 + std::error_code ec; + for (const auto &entry : std::filesystem::directory_iterator(path, ec)) { + if (!entry.is_directory()) { + ret[entry.path().filename().string()] = util::read_file(entry.path().string()); + } + } +#else DIR *d = opendir(path.c_str()); if (!d) return ret; @@ -113,6 +135,7 @@ std::map read_files_in_dir(const std::string &path) { } closedir(d); +#endif return ret; } @@ -152,6 +175,7 @@ int safe_fflush(FILE *stream) { return ret; } +#ifndef _WIN32 int safe_ioctl(int fd, unsigned long request, void *argp, const char* exception_msg) { int ret; do { @@ -164,8 +188,13 @@ int safe_ioctl(int fd, unsigned long request, void *argp, const char* exception_ } return ret; } +#endif std::string readlink(const std::string &path) { +#ifdef _WIN32 + (void)path; + return ""; // no symlinks are created on Windows +#else char buff[4096]; ssize_t len = ::readlink(path.c_str(), buff, sizeof(buff)-1); if (len != -1) { @@ -173,6 +202,7 @@ std::string readlink(const std::string &path) { return std::string(buff); } return ""; +#endif } bool file_exists(const std::string& fn) { @@ -285,6 +315,31 @@ std::string strip(const std::string &str) { return str.substr(start, end - start + 1); } +int lock_file_exclusive(int fd) { +#ifdef _WIN32 + OVERLAPPED ov = {}; + return LockFileEx((HANDLE)_get_osfhandle(fd), LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ov) ? 0 : -1; +#else + return HANDLE_EINTR(flock(fd, LOCK_EX)); +#endif +} + +int replace_file(const char *from, const char *to) { +#ifdef _WIN32 + // readers open files without FILE_SHARE_DELETE, so replacing a param that another process is reading fails with a + // sharing violation; reads take microseconds, so wait them out instead of dropping the write + for (int i = 0; i < 200; ++i) { + if (MoveFileExA(from, to, MOVEFILE_REPLACE_EXISTING)) return 0; + DWORD err = GetLastError(); + if (err != ERROR_SHARING_VIOLATION && err != ERROR_ACCESS_DENIED) break; + Sleep(1); + } + return -1; +#else + return rename(from, to); +#endif +} + std::string check_output(const std::string& command) { char buffer[128]; std::string result; diff --git a/openpilot/common/util.h b/openpilot/common/util.h index e4483ee7a57c4e..ec3a2738a83663 100644 --- a/openpilot/common/util.h +++ b/openpilot/common/util.h @@ -4,6 +4,39 @@ #include #include +#ifdef _WIN32 +#include +#include +#include +#include +#include +#include +#include +// POSIX calls this codebase uses that the Windows CRT spells differently +inline int mkdir(const char *path, mode_t) { return _mkdir(path); } +inline int setenv(const char *name, const char *value, int) { return _putenv_s(name, value); } +inline int unsetenv(const char *name) { return _putenv_s(name, ""); } +inline struct tm *localtime_r(const time_t *t, struct tm *out) { + struct tm *r = localtime(t); // thread-local storage in the Windows CRT + if (r) *out = *r; + return r ? out : nullptr; +} +inline time_t timegm(struct tm *tm) { return _mkgmtime(tm); } +inline char *strptime(const char *s, const char *format, struct tm *tm) { + std::istringstream in(s); + in >> std::get_time(tm, format); + if (in.fail()) return nullptr; + return const_cast(s) + (in.eof() ? strlen(s) : static_cast(in.tellg())); +} +// popen/pclose return the exit code directly, there is no wait status to decode +#define WIFEXITED(status) 1 +#define WEXITSTATUS(status) (status) +#define WIFSIGNALED(status) 0 +#define WTERMSIG(status) 0 +#endif + +#include "common/file.h" + #include #include #include @@ -119,7 +152,9 @@ class ExitHandler { std::signal(SIGINT, (sighandler_t)set_do_exit); std::signal(SIGTERM, (sighandler_t)set_do_exit); -#ifndef __APPLE__ +#ifdef _WIN32 + std::signal(SIGBREAK, (sighandler_t)set_do_exit); // CTRL_BREAK_EVENT from the manager +#elif !defined(__APPLE__) std::signal(SIGPWR, (sighandler_t)set_do_exit); #endif } @@ -133,7 +168,7 @@ class ExitHandler { } private: static void set_do_exit(int sig) { -#ifndef __APPLE__ +#if !defined(__APPLE__) && !defined(_WIN32) power_failure = (sig == SIGPWR); #endif signal = sig; diff --git a/openpilot/common/win32.h b/openpilot/common/win32.h new file mode 100644 index 00000000000000..6b849a38882d13 --- /dev/null +++ b/openpilot/common/win32.h @@ -0,0 +1,20 @@ +#pragma once + +// Include this instead of . It keeps the Win32 headers lean and drops the +// macros that collide with identifiers in the capnp schemas. Never include it from a +// header, and never in a translation unit that also sees common/params.h: its BOOL, +// INT and FLOAT enumerators clash with the Win32 typedefs. +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef NOGDI +#define NOGDI +#endif +#include +#undef NO_ERROR +#undef MessageBox +#endif diff --git a/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript b/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript index fa249765bc3cbf..7a3a12b85558b8 100644 --- a/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript +++ b/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript @@ -63,11 +63,11 @@ source_list = ['long_mpc.py', lenv = env.Clone() copied_acados_libs = [] -if arch != "Darwin": +if arch not in ("Darwin", "Windows"): # the Windows acados wheel ships static archives, they link into the solver library for lib in ["libacados.so", "libblasfeo.so", "libhpipm.so", "libqpOASES_e.so.3.1"]: copied_acados_libs += lenv.Command(f"{gen}/{lib}", Dir(acados.LIB_DIR).File(lib), [Mkdir(Dir(gen)), Copy("$TARGET", "$SOURCE")]) lenv["RPATH"] += [lenv.Literal('\\$$ORIGIN')] -else: +elif arch == "Darwin": acados_rel_path = Dir(gen).rel_path(Dir(acados.LIB_DIR)) lenv["RPATH"] += [lenv.Literal(f'\\$$ORIGIN/{acados_rel_path}')] lenv.Clean(generated_files, Dir(gen)) @@ -79,9 +79,9 @@ lenv.Depends(generated_long, [msgq_python, common_python]) lenv["CFLAGS"].append("-DACADOS_WITH_QPOASES") lenv["CXXFLAGS"].append("-DACADOS_WITH_QPOASES") lenv["CCFLAGS"].append("-Wno-unused") -if arch != "Darwin": +if arch not in ("Darwin", "Windows"): lenv["LINKFLAGS"].append("-Wl,--disable-new-dtags") -else: +elif arch == "Darwin": lenv["LINKFLAGS"].append("-Wl,-install_name,@loader_path/libacados_ocp_solver_long.dylib") lenv["LINKFLAGS"].append(f"-Wl,-rpath,@loader_path/{acados_rel_path}") lib_solver = lenv.SharedLibrary(f"{gen}/acados_ocp_solver_long", @@ -100,11 +100,11 @@ lenv2["RPATH"] += [lenv2.Literal('\\$$ORIGIN')] lenv2.Command(libacados_ocp_solver_c, [acados_ocp_solver_pyx, acados_ocp_solver_common, libacados_ocp_solver_pxd], f'cython' + \ - f' -o {libacados_ocp_solver_c.get_labspath()}' + \ - f' -I {libacados_ocp_solver_pxd.get_dir().get_labspath()}' + \ - f' -I {acados_ocp_solver_common.get_dir().get_labspath()}' + \ - f' {acados_ocp_solver_pyx.get_labspath()}') -lib_cython = lenv2.Program(f'{gen}/acados_ocp_solver_pyx.so', [libacados_ocp_solver_c], LIBS=['acados_ocp_solver_long']) + f' -o {libacados_ocp_solver_c.abspath}' + \ + f' -I {libacados_ocp_solver_pxd.get_dir().abspath}' + \ + f' -I {acados_ocp_solver_common.get_dir().abspath}' + \ + f' {acados_ocp_solver_pyx.abspath}') +lib_cython = lenv2.Program(f'{gen}/acados_ocp_solver_pyx.so', [libacados_ocp_solver_c], LIBS=lenv2["LIBS"] + ['acados_ocp_solver_long']) lenv2.Depends(lib_cython, lib_solver) lenv2.Depends(lib_cython, copied_acados_libs) lenv2.Depends(libacados_ocp_solver_c, np_version) diff --git a/openpilot/selfdrive/locationd/test/test_locationd_scenarios.py b/openpilot/selfdrive/locationd/test/test_locationd_scenarios.py index 01fc3f11773172..e32276388638bc 100644 --- a/openpilot/selfdrive/locationd/test/test_locationd_scenarios.py +++ b/openpilot/selfdrive/locationd/test/test_locationd_scenarios.py @@ -1,7 +1,11 @@ -import fcntl import numpy as np import os +import sys import tempfile +try: + import fcntl +except ImportError: # Windows + import msvcrt from collections import defaultdict from enum import Enum @@ -100,6 +104,16 @@ def timing_spike(msg): return get_select_fields_data(logs), get_select_fields_data(replayed_logs) +def lock_exclusive(f): + if sys.platform == "win32": + while True: + try: + return msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) # gives up after 10 s, keep waiting + except OSError: + pass + fcntl.flock(f, fcntl.LOCK_EX) + + class TestLocationdScenarios(OpenpilotTestCase): """ Test locationd with different scenarios. In all these scenarios, we expect the following: @@ -116,7 +130,7 @@ def setup_class(cls): ready_path = f"{lock_path}.ready" logs = None with open(lock_path, "w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) + lock_exclusive(lock) if not os.path.exists(ready_path): logs = list(LogReader(TEST_ROUTE)) open(ready_path, "w").close() diff --git a/openpilot/selfdrive/modeld/SConscript b/openpilot/selfdrive/modeld/SConscript index af10467529ec7b..5221de49a66973 100644 --- a/openpilot/selfdrive/modeld/SConscript +++ b/openpilot/selfdrive/modeld/SConscript @@ -32,7 +32,7 @@ if arch == 'comma_arm64': else: camera_configs = [(c.width, c.height) for c in (_ar_ox_fisheye, _os_fisheye)] tg_backend = 'CPU' - tg_flags = f'DEV=CPU' if arch == 'Darwin' else 'DEV=CPU:LLVM' + tg_flags = f'DEV=CPU' if arch in ('Darwin', 'Windows') else 'DEV=CPU:LLVM' tg_devices = { # which device to put jit inputs to at runtime 'openpilot.selfdrive.modeld.dmonitoringmodeld': { @@ -124,17 +124,19 @@ for cam_w, cam_h in camera_configs: lenv.Command(dm_pkl_path, tinygrad_files + compile_dm_warp_script + compile_modeld_script + [tg_devices_node], cmd) def tg_compile(flags, model_name): - pythonpath_string = 'PYTHONPATH="${PYTHONPATH}:' + env.Dir("#tinygrad_repo").abspath + '"' + pythonpath_string = 'PYTHONPATH="${PYTHONPATH}' + os.pathsep + env.Dir("#tinygrad_repo").abspath + '"' fn = File(f"models/{model_name}").abspath pkl = fn + "_tinygrad.pkl" onnx_path = fn + ".onnx" + # tinygrad's fetch() only takes paths starting with "/" or "." as local files, so give it a Windows drive path relative to the cwd + onnx_arg = onnx_path if arch != 'Windows' else "./" + os.path.relpath(onnx_path, env.Dir("#").abspath).replace(os.sep, "/") chunk_targets = get_chunk_targets(pkl, estimate_pickle_max_size(os.path.getsize(onnx_path))) def do_chunk(target, source, env): chunk_file(pkl, chunk_targets) return lenv.Command( chunk_targets, [onnx_path] + tinygrad_files + [Value(chunk_targets), chunker_file, tg_devices_node], - [f'{pythonpath_string} {flags} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {pkl}', + [f'{pythonpath_string} {flags} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {onnx_arg} {pkl}', Action(do_chunk, " [CHUNK] $TARGET")], ) diff --git a/openpilot/selfdrive/pandad/SConscript b/openpilot/selfdrive/pandad/SConscript index fd59db98537941..afb931b4b51ee0 100644 --- a/openpilot/selfdrive/pandad/SConscript +++ b/openpilot/selfdrive/pandad/SConscript @@ -1,6 +1,6 @@ Import('env', 'arch', 'common', 'messaging') -if arch != "Darwin": +if arch not in ("Darwin", "Windows"): libs = [common, messaging, 'pthread'] panda = env.Library('panda', ['panda.cc', 'spi.cc']) diff --git a/openpilot/selfdrive/test/process_replay/model_replay.py b/openpilot/selfdrive/test/process_replay/model_replay.py index 927c9b38f1f892..2f10c465c067ea 100755 --- a/openpilot/selfdrive/test/process_replay/model_replay.py +++ b/openpilot/selfdrive/test/process_replay/model_replay.py @@ -13,6 +13,7 @@ from openpilot.common.git import get_commit from openpilot.common.hardware import PC +from openpilot.common.hardware.hw import TMP_DIR from openpilot.tools.lib.openpilotci import get_url from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff from openpilot.selfdrive.test.process_replay.process_replay import get_process_config, replay_process @@ -200,10 +201,10 @@ def model_replay(lr, frs): def get_frames(): regen_cache = "--regen-cache" in sys.argv - frames_cache = '/tmp/model_replay_cache' if PC else '/data/model_replay_cache' + frames_cache = os.path.join(TMP_DIR, 'model_replay_cache') if PC else '/data/model_replay_cache' os.makedirs(frames_cache, exist_ok=True) - cache_name = f'{frames_cache}/{TEST_ROUTE}_{SEGMENT}_{START_FRAME}_{END_FRAME}.pkl' + cache_name = f'{frames_cache}/{TEST_ROUTE.replace("|", "_")}_{SEGMENT}_{START_FRAME}_{END_FRAME}.pkl' # no '|' in Windows file names if os.path.isfile(cache_name) and not regen_cache: try: print(f"Loading frames from cache {cache_name}") diff --git a/openpilot/selfdrive/test/process_replay/process_replay.py b/openpilot/selfdrive/test/process_replay/process_replay.py index 5abfab2c35aa73..6284fd23d8c7ce 100755 --- a/openpilot/selfdrive/test/process_replay/process_replay.py +++ b/openpilot/selfdrive/test/process_replay/process_replay.py @@ -3,7 +3,6 @@ import time import copy import heapq -import signal import numpy as np from collections import Counter from dataclasses import dataclass, field @@ -25,6 +24,7 @@ from openpilot.common.timeout import Timeout from openpilot.common.realtime import DT_CTRL from openpilot.system.camerad.cameras.nv12_info import get_nv12_info +from openpilot.system.manager.process import SIGKILL from openpilot.system.manager.process_config import managed_processes from openpilot.selfdrive.test.process_replay.vision_meta import meta_from_camera_state, available_streams from openpilot.selfdrive.test.process_replay.migration import migrate_all @@ -252,7 +252,7 @@ def start( def stop(self): with self.prefix: - self.process.signal(signal.SIGKILL) + self.process.signal(SIGKILL) self.process.stop() self.rc.close_context() self.prefix.clean_dirs() diff --git a/openpilot/system/athena/athenad.py b/openpilot/system/athena/athenad.py index 2b676387762844..d5ee86c68a71db 100755 --- a/openpilot/system/athena/athenad.py +++ b/openpilot/system/athena/athenad.py @@ -387,10 +387,10 @@ def scan_dir(path: str, prefix: str) -> list[str]: # (glob and friends traverse entire dir tree) with os.scandir(path) as i: for e in i: - rel_path = os.path.relpath(e.path, Paths.log_root()) + rel_path = os.path.relpath(e.path, Paths.log_root()).replace(os.sep, '/') if e.is_dir(follow_symlinks=False): # add trailing slash - rel_path = os.path.join(rel_path, '') + rel_path += '/' # if prefix is a partial dir name, current dir will start with prefix # if prefix is a partial file name, prefix with start with dir name if rel_path.startswith(prefix) or prefix.startswith(rel_path): diff --git a/openpilot/system/athena/tests/helpers.py b/openpilot/system/athena/tests/helpers.py index dbca66be10cb35..65333386fb6094 100644 --- a/openpilot/system/athena/tests/helpers.py +++ b/openpilot/system/athena/tests/helpers.py @@ -1,3 +1,4 @@ +import contextlib import http.server import socket @@ -30,7 +31,8 @@ def run(self): finally: conn.shutdown(0) conn.close() - self.socket.shutdown(0) + with contextlib.suppress(OSError): # Windows refuses to shut down a listening socket + self.socket.shutdown(0) self.socket.close() @@ -46,7 +48,15 @@ class MockWebsocket: def __init__(self, recv_queue, send_queue): self.recv_queue = recv_queue self.send_queue = send_queue - self.sock = socket.socket() + # the proxy selects on this socket before calling recv() and sets IP_TOS on it, so it has to be a readable TCP + # socket: a connected loopback pair with a byte in flight (only Linux reports an unconnected socket readable) + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.bind(('127.0.0.1', 0)) + listener.listen(1) + self._peer = socket.create_connection(listener.getsockname()) + self.sock, _ = listener.accept() + listener.close() + self._peer.send(b'x') def recv(self): data = self.recv_queue.get() @@ -59,6 +69,7 @@ def send(self, data, opcode): def close(self): self.sock.close() + self._peer.close() class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler): diff --git a/openpilot/system/athena/tests/test_athenad.py b/openpilot/system/athena/tests/test_athenad.py index f9e4c0d3341303..b3fd0c49735560 100644 --- a/openpilot/system/athena/tests/test_athenad.py +++ b/openpilot/system/athena/tests/test_athenad.py @@ -55,6 +55,16 @@ def host(): with http_server_context(handler=HTTPRequestHandler, setup=seed_athena_server) as (host, port): yield f"http://{host}:{port}" +def send_device_state(started, end_event): + # module level: Windows spawns the process and cannot pickle a closure or a socket + pub_sock = messaging.pub_sock("deviceState") + started.set() + while not end_event.is_set(): + msg = messaging.new_message('deviceState') + pub_sock.send(msg.to_bytes()) + time.sleep(0.01) + + class TestAthenadMethods(OpenpilotTestCase): @classmethod def setup_class(cls): @@ -121,18 +131,11 @@ def test_get_message(self): with self.assertRaises(TimeoutError) as _: dispatcher["getMessage"]("controlsState") - end_event = multiprocessing.Event() - - pub_sock = messaging.pub_sock("deviceState") + started, end_event = multiprocessing.Event(), multiprocessing.Event() - def send_deviceState(): - while not end_event.is_set(): - msg = messaging.new_message('deviceState') - pub_sock.send(msg.to_bytes()) - time.sleep(0.01) - - p = multiprocessing.Process(target=send_deviceState) + p = multiprocessing.Process(target=send_device_state, args=(started, end_event)) p.start() + assert started.wait(10) # spawning the process takes a while on Windows time.sleep(0.1) try: deviceState = dispatcher["getMessage"]("deviceState") @@ -326,7 +329,10 @@ def test_upload_handler_timeout(self): athenad.upload_queue.put_nowait(item) self._wait_for_upload() - time.sleep(0.1) + for _ in range(100): # a refused connection takes a few seconds on Windows + if athenad.upload_queue.qsize(): + break + time.sleep(0.1) # Check that upload item was put back in the queue with incremented retry count assert athenad.upload_queue.qsize() == 1 @@ -350,7 +356,7 @@ def test_cancel_upload(self): @with_upload_handler def test_cancel_expiry(self): t_future = datetime.now() - timedelta(days=40) - ts = int(t_future.strftime("%s")) * 1000 + ts = int(t_future.timestamp()) * 1000 # Item that would time out if actually uploaded fn = self._create_file('qlog.zst') diff --git a/openpilot/system/camerad/webcam/camerad.py b/openpilot/system/camerad/webcam/camerad.py index 4dfa5f33711ed0..a5961d4f6a941b 100755 --- a/openpilot/system/camerad/webcam/camerad.py +++ b/openpilot/system/camerad/webcam/camerad.py @@ -32,7 +32,7 @@ def __init__(self): self.cameras = [] for c in CAMERAS: - cam_device = f"/dev/video{c.cam_id}" if platform.system() != "Darwin" else c.cam_id + cam_device = f"/dev/video{c.cam_id}" if platform.system() == "Linux" else c.cam_id cam = Camera(c.msg_name, c.stream_type, cam_device) self.cameras.append(cam) self.vipc_server.create_buffers(c.stream_type, 20, cam.W, cam.H) diff --git a/openpilot/system/hardware/hardwared.py b/openpilot/system/hardware/hardwared.py index 11fe41400d6fd8..3c5fb3c3acc6ba 100755 --- a/openpilot/system/hardware/hardwared.py +++ b/openpilot/system/hardware/hardwared.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -import fcntl import os import queue import struct @@ -115,6 +114,7 @@ def set_offroad_alert_if_changed(offroad_alert: str, show_alert: bool, extra_tex set_offroad_alert(offroad_alert, show_alert, extra_text) def touch_thread(end_event): + import fcntl # POSIX only; this thread reads the device's touch input, so keep the module importable on Windows count = 0 pm = messaging.PubMaster(["touch"]) diff --git a/openpilot/system/loggerd/bootlog.cc b/openpilot/system/loggerd/bootlog.cc index b2e3abfcf1264b..2f55590605311b 100644 --- a/openpilot/system/loggerd/bootlog.cc +++ b/openpilot/system/loggerd/bootlog.cc @@ -42,7 +42,7 @@ static kj::Array build_boot_log() { lentry.setValue(capnp::Data::Reader((const kj::byte*)result.data(), result.size())); } - boot.setLaunchLog(util::read_file("/tmp/launch_log")); + boot.setLaunchLog(util::read_file(Path::tmp_dir() + "/launch_log")); return capnp::messageToFlatArray(msg); } diff --git a/openpilot/system/loggerd/config.py b/openpilot/system/loggerd/config.py index c2d213e90617a8..2f1edc286a5b13 100644 --- a/openpilot/system/loggerd/config.py +++ b/openpilot/system/loggerd/config.py @@ -1,14 +1,23 @@ import os +import shutil from openpilot.common.hardware.hw import Paths CAMERA_FPS = 20 SEGMENT_LENGTH = 60 +def _free_and_total_bytes(path: str) -> tuple[int, int]: + if statvfs := getattr(os, "statvfs", None): + st = statvfs(path) + return st.f_bavail * st.f_frsize, st.f_blocks * st.f_frsize + usage = shutil.disk_usage(path) # Windows has no statvfs + return usage.free, usage.total + + def get_available_percent(default: float) -> float: try: - statvfs = os.statvfs(Paths.log_root()) - available_percent = 100.0 * statvfs.f_bavail / statvfs.f_blocks + free, total = _free_and_total_bytes(Paths.log_root()) + available_percent = 100.0 * free / total except OSError: available_percent = default @@ -17,8 +26,7 @@ def get_available_percent(default: float) -> float: def get_available_bytes(default: int) -> int: try: - statvfs = os.statvfs(Paths.log_root()) - available_bytes = statvfs.f_bavail * statvfs.f_frsize + available_bytes, _ = _free_and_total_bytes(Paths.log_root()) except OSError: available_bytes = default diff --git a/openpilot/system/loggerd/loggerd.cc b/openpilot/system/loggerd/loggerd.cc index af4a5c0cde04a8..ae5d9c213ec63c 100644 --- a/openpilot/system/loggerd/loggerd.cc +++ b/openpilot/system/loggerd/loggerd.cc @@ -1,4 +1,6 @@ +#ifndef _WIN32 #include +#endif #include #include @@ -203,6 +205,10 @@ void handle_preserve_segment(LoggerdState *s) { #ifdef __APPLE__ int ret = setxattr(s->logger.segmentPath().c_str(), PRESERVE_ATTR_NAME, &PRESERVE_ATTR_VALUE, 1, 0, 0); +#elif defined(_WIN32) + // NTFS alternate data stream, the same place xattr_cache.py reads + std::ofstream stream(s->logger.segmentPath() + ":" + PRESERVE_ATTR_NAME, std::ios::binary); + int ret = stream.write(&PRESERVE_ATTR_VALUE, 1).good() ? 0 : -1; #else int ret = setxattr(s->logger.segmentPath().c_str(), PRESERVE_ATTR_NAME, &PRESERVE_ATTR_VALUE, 1, 0); #endif @@ -334,7 +340,9 @@ void loggerd_thread() { if (do_exit.power_failure) { LOGE("power failure"); +#ifndef _WIN32 sync(); +#endif LOGE("sync done"); } diff --git a/openpilot/system/loggerd/tests/test_loggerd.py b/openpilot/system/loggerd/tests/test_loggerd.py index de91cae1dbb1f4..c7b5bb01835452 100644 --- a/openpilot/system/loggerd/tests/test_loggerd.py +++ b/openpilot/system/loggerd/tests/test_loggerd.py @@ -17,7 +17,7 @@ from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params from openpilot.common.timeout import Timeout -from openpilot.common.hardware.hw import Paths +from openpilot.common.hardware.hw import Paths, TMP_DIR from openpilot.common.hardware import COMMA_HARDWARE from openpilot.system.loggerd.xattr_cache import getxattr from openpilot.system.loggerd.deleter import PRESERVE_ATTR_NAME, PRESERVE_ATTR_VALUE @@ -57,7 +57,8 @@ def _get_log_fn(self, x): def _gen_bootlog(self): with Timeout(5): - out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "openpilot/system/loggerd"), encoding='utf-8') + loggerd_dir = os.path.join(BASEDIR, "openpilot/system/loggerd") + out = subprocess.check_output(os.path.join(loggerd_dir, "bootlog"), cwd=loggerd_dir, encoding='utf-8') log_fn = self._get_log_fn(out) @@ -215,7 +216,7 @@ def test_rotation(self): def test_bootlog(self): # generate bootlog with fake launch log launch_log = ''.join(str(random.choice(string.printable)) for _ in range(100)) - with open("/tmp/launch_log", "w") as f: + with open(os.path.join(TMP_DIR, "launch_log"), "w", newline="") as f: # no newline translation on Windows f.write(launch_log) bootlog_path = self._gen_bootlog() diff --git a/openpilot/system/loggerd/tests/test_uploader.py b/openpilot/system/loggerd/tests/test_uploader.py index b83b3569e2831f..62d938195afb01 100644 --- a/openpilot/system/loggerd/tests/test_uploader.py +++ b/openpilot/system/loggerd/tests/test_uploader.py @@ -1,4 +1,3 @@ -import os import threading import logging import json @@ -7,6 +6,7 @@ from openpilot.common.swaglog import cloudlog from openpilot.system.loggerd.uploader import clear_locks, main, Uploader, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE +from openpilot.system.loggerd.xattr_cache import getxattr from openpilot.system.loggerd.tests.loggerd_tests_common import UploaderTestCase @@ -88,7 +88,7 @@ def test_upload(self): assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload" assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice" for f_path in exp_order: - assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded" + assert getxattr(str((Path(Paths.log_root()) / f_path).with_suffix("")), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded" assert log_handler.upload_order == exp_order, "Files uploaded in wrong order" @@ -104,7 +104,7 @@ def test_upload_with_wrong_xattr(self): assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload" assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice" for f_path in exp_order: - assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded" + assert getxattr(str((Path(Paths.log_root()) / f_path).with_suffix("")), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded" assert log_handler.upload_order == exp_order, "Files uploaded in wrong order" @@ -121,7 +121,7 @@ def test_upload_ignored(self): assert not len(log_handler.upload_ignored) < len(exp_order), "Some files failed to ignore" assert not len(log_handler.upload_ignored) > len(exp_order), "Some files were ignored twice" for f_path in exp_order: - assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not ignored" + assert getxattr(str((Path(Paths.log_root()) / f_path).with_suffix("")), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not ignored" assert log_handler.upload_ignored == exp_order, "Files ignored in wrong order" @@ -145,7 +145,7 @@ def test_upload_files_in_create_order(self): assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload" assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice" for f_path in exp_order: - assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded" + assert getxattr(str((Path(Paths.log_root()) / f_path).with_suffix("")), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded" assert log_handler.upload_order == exp_order, "Files uploaded in wrong order" diff --git a/openpilot/system/loggerd/uploader.py b/openpilot/system/loggerd/uploader.py index 81f8ed1ba3c837..e8da31a0fabbf8 100755 --- a/openpilot/system/loggerd/uploader.py +++ b/openpilot/system/loggerd/uploader.py @@ -100,7 +100,7 @@ def list_upload_files(self, metered: bool) -> Iterator[tuple[str, str, str]]: continue for name in sorted(names, key=lambda n: self.immediate_priority.get(n, 1000)): - key = os.path.join(logdir, name) + key = f"{logdir}/{name}" # the upload key, not a local path fn = os.path.join(path, name) # skip files already uploaded try: @@ -128,7 +128,7 @@ def next_file_to_upload(self, metered: bool) -> tuple[str, str, str] | None: upload_files = list(self.list_upload_files(metered)) for name, key, fn in upload_files: - if any(f in fn for f in self.immediate_folders): + if any(key.startswith(f) for f in self.immediate_folders): return name, key, fn for name, key, fn in upload_files: diff --git a/openpilot/system/loggerd/xattr_cache.py b/openpilot/system/loggerd/xattr_cache.py index e0f6ba9588a938..4cf8d8243c9015 100644 --- a/openpilot/system/loggerd/xattr_cache.py +++ b/openpilot/system/loggerd/xattr_cache.py @@ -18,6 +18,12 @@ def _raise_os_error(path: str) -> None: def _getxattr(path: str, attr_name: str) -> bytes: + if sys.platform == "win32": + try: + with open(f"{path}:{attr_name}", "rb") as f: # NTFS alternate data stream + return f.read() + except FileNotFoundError: + raise OSError(errno.ENODATA, os.strerror(errno.ENODATA), path) from None if sys.platform != "darwin": return os.getxattr(path, attr_name) @@ -39,6 +45,10 @@ def _getxattr(path: str, attr_name: str) -> bytes: def _setxattr(path: str, attr_name: str, attr_value: bytes) -> None: + if sys.platform == "win32": + with open(f"{path}:{attr_name}", "wb") as f: + f.write(attr_value) + return if sys.platform != "darwin": os.setxattr(path, attr_name, attr_value) return diff --git a/openpilot/system/manager/helpers.py b/openpilot/system/manager/helpers.py index 453e13184de911..79790eab3d44f1 100644 --- a/openpilot/system/manager/helpers.py +++ b/openpilot/system/manager/helpers.py @@ -1,5 +1,4 @@ import errno -import fcntl import os import sys import pathlib @@ -13,6 +12,10 @@ from openpilot.common.params import Params def unblock_stdout() -> None: + if sys.platform == "win32": + return # no pty or fork on Windows; the console does not block the manager + import fcntl # POSIX only, keep the module importable on Windows + # get a non-blocking stdout child_pid, child_pty = os.forkpty() if child_pid != 0: # parent @@ -54,7 +57,8 @@ def save_bootlog(): def fn(tmpdir): env = os.environ.copy() env['PARAMS_COPY_PATH'] = tmpdir - subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "openpilot/system/loggerd"), env=env) + # absolute path: CreateProcess resolves "./bootlog" against the parent's cwd, not the cwd argument + subprocess.call(os.path.join(BASEDIR, "openpilot/system/loggerd/bootlog"), cwd=os.path.join(BASEDIR, "openpilot/system/loggerd"), env=env) shutil.rmtree(tmpdir) t = threading.Thread(target=fn, args=(tmp, )) t.daemon = True diff --git a/openpilot/system/manager/process.py b/openpilot/system/manager/process.py index 363f7f38e7e28c..9ee59eb09da75c 100644 --- a/openpilot/system/manager/process.py +++ b/openpilot/system/manager/process.py @@ -1,6 +1,7 @@ import importlib import os import signal +import sys import time import subprocess from collections.abc import Callable, ValuesView @@ -16,6 +17,8 @@ from openpilot.common.params import Params from openpilot.common.swaglog import cloudlog +SIGKILL = getattr(signal, "SIGKILL", signal.SIGTERM) # Windows: os.kill() terminates the process either way + def launcher(proc: str, name: str) -> None: try: @@ -42,6 +45,35 @@ def launcher(proc: str, name: str) -> None: raise +class PopenProcess: + """The multiprocessing.Process view of a subprocess.Popen: os.execvp() cannot replace a process on Windows.""" + + def __init__(self, pargs: list[str], cwd: str, name: str): + self.name = name + if os.sep in pargs[0] or "/" in pargs[0]: + pargs = [os.path.join(cwd, pargs[0]), *pargs[1:]] # CreateProcess resolves relative paths against the parent + env = {**os.environ, "MANAGER_DAEMON": name} + # its own process group so that CTRL_BREAK_EVENT reaches only this process (SIGBREAK in the C++ ExitHandler) + self._popen = subprocess.Popen(pargs, cwd=cwd, env=env, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + + @property + def pid(self) -> int: + return self._popen.pid + + @property + def exitcode(self) -> int | None: + return self._popen.poll() + + def is_alive(self) -> bool: + return self._popen.poll() is None + + def join(self, timeout: float | None = None) -> None: + try: + self._popen.wait(timeout) + except subprocess.TimeoutExpired: + pass + + def nativelauncher(pargs: list[str], cwd: str, name: str) -> None: os.environ['MANAGER_DAEMON'] = name @@ -50,7 +82,7 @@ def nativelauncher(pargs: list[str], cwd: str, name: str) -> None: os.execvp(pargs[0], pargs) -def join_process(process: Process, timeout: float) -> None: +def join_process(process: "Process | PopenProcess", timeout: float) -> None: # Process().join(timeout) will hang due to a python 3 bug: https://bugs.python.org/issue28382 # We have to poll the exitcode instead t = time.monotonic() @@ -62,7 +94,7 @@ class ManagerProcess(ABC): daemon = False sigkill = False should_run: Callable[[bool, Params, car.CarParams], bool] - proc: Process | None = None + proc: Process | PopenProcess | None = None enabled = True name = "" shutting_down = False @@ -79,7 +111,7 @@ def stop(self, retry: bool = True, block: bool = True, sig: signal.Signals | Non if not self.shutting_down: cloudlog.info(f"killing {self.name}") if sig is None: - sig = signal.SIGKILL if self.sigkill else signal.SIGINT + sig = SIGKILL if self.sigkill else signal.SIGINT self.signal(sig) self.shutting_down = True @@ -91,7 +123,7 @@ def stop(self, retry: bool = True, block: bool = True, sig: signal.Signals | Non # If process failed to die send SIGKILL if self.proc.exitcode is None and retry: cloudlog.info(f"killing {self.name} with SIGKILL") - self.signal(signal.SIGKILL) + self.signal(SIGKILL) self.proc.join() ret = self.proc.exitcode @@ -116,7 +148,16 @@ def signal(self, sig: int) -> None: return cloudlog.info(f"sending signal {sig} to {self.name}") - os.kill(self.proc.pid, sig) + if sys.platform == "win32" and isinstance(self.proc, PopenProcess) and sig != SIGKILL: + sig = signal.CTRL_BREAK_EVENT # os.kill() terminates for any other signal; this one lets the daemon exit cleanly + # Python daemons are multiprocessing children in the manager's own console group, which no console event can + # target, so on Windows os.kill() terminates them outright (no KeyboardInterrupt cleanup): accepted for development + try: + os.kill(self.proc.pid, sig) + except PermissionError: + if sys.platform != "win32": + raise + cloudlog.info(f"{self.name} is already terminating") # Windows refuses to open a process that is going away def get_process_state_msg(self): state = log.ManagerState.ProcessState.new_message() @@ -149,8 +190,11 @@ def start(self) -> None: cwd = os.path.join(BASEDIR, self.cwd) cloudlog.info(f"starting process {self.name}") - self.proc = Process(name=self.name, target=self.launcher, args=(self.cmdline, cwd, self.name)) - self.proc.start() + if sys.platform == "win32": + self.proc = PopenProcess(self.cmdline, cwd, self.name) + else: + self.proc = Process(name=self.name, target=self.launcher, args=(self.cmdline, cwd, self.name)) + self.proc.start() self.shutting_down = False diff --git a/openpilot/system/manager/process_config.py b/openpilot/system/manager/process_config.py index b8a1e4a12e9df1..ddf91b13326d0b 100644 --- a/openpilot/system/manager/process_config.py +++ b/openpilot/system/manager/process_config.py @@ -77,8 +77,8 @@ def and_(*fns): NativeProcess("camerad", "openpilot/system/camerad", ["./camerad"], or_(driverview, livestream), enabled=not WEBCAM), PythonProcess("webcamerad", "openpilot.system.camerad.webcam.camerad", driverview, enabled=WEBCAM), - PythonProcess("proclogd", "openpilot.system.proclogd", only_onroad, enabled=platform.system() != "Darwin"), - PythonProcess("journald", "openpilot.system.journald", only_onroad, platform.system() != "Darwin"), + PythonProcess("proclogd", "openpilot.system.proclogd", only_onroad, enabled=platform.system() == "Linux"), + PythonProcess("journald", "openpilot.system.journald", only_onroad, platform.system() == "Linux"), PythonProcess("micd", "openpilot.system.micd", iscar), PythonProcess("timed", "openpilot.system.timed", always_run, enabled=not PC), diff --git a/openpilot/system/tests/test_logmessaged.py b/openpilot/system/tests/test_logmessaged.py index 247bfd8a429606..be7fde0c03a47d 100644 --- a/openpilot/system/tests/test_logmessaged.py +++ b/openpilot/system/tests/test_logmessaged.py @@ -19,7 +19,12 @@ def setup_method(self): self.sock = messaging.sub_sock("logMessage", timeout=1000, conflate=False) self.error_sock = messaging.sub_sock("logMessage", timeout=1000, conflate=False) - # ensure sockets are connected + # ensure sockets are connected and the daemon is up (spawning it takes a while on Windows) + for _ in range(100): + cloudlog.error("logmessaged ready check") + time.sleep(0.1) + if messaging.drain_sock(self.sock): + break time.sleep(0.5) messaging.drain_sock(self.sock) messaging.drain_sock(self.error_sock) @@ -32,6 +37,9 @@ def teardown_method(self): def _get_log_files(self): return list(glob.glob(os.path.join(Paths.swaglog_root(), "swaglog.*"))) + def _log_size(self): + return sum(os.path.getsize(f) for f in self._get_log_files()) + def test_simple_log(self): msgs = [f"abc {i}" for i in range(10)] for m in msgs: @@ -44,12 +52,16 @@ def test_simple_log(self): def test_big_log(self): n = 10 msg = "a"*3*1024*1024 + base = self._log_size() # the ready checks from setup_method are in the same files for _ in range(n): cloudlog.info(msg) - time.sleep(0.5) + for _ in range(300): # writing 30 MB takes a while on a loaded machine + time.sleep(0.1) + logsize = self._log_size() - base + if logsize > n * len(msg): + break msgs = messaging.drain_sock(self.sock) assert len(msgs) == 0 - logsize = sum([os.path.getsize(f) for f in self._get_log_files()]) assert (n*len(msg)) < logsize < (n*(len(msg)+1024)) diff --git a/openpilot/system/ui/lib/application.py b/openpilot/system/ui/lib/application.py index 230ddc635e2d9e..c92f48ef9b7824 100644 --- a/openpilot/system/ui/lib/application.py +++ b/openpilot/system/ui/lib/application.py @@ -769,6 +769,8 @@ def _begin_scissor_mode_scaled(x, y, width, height): rl.begin_scissor_mode = _begin_scissor_mode_scaled def _set_log_callback(self): + if sys.platform == "win32": + return # no vasprintf in the Windows CRT; raylib keeps its default stdout logging ffi_libc = cffi.FFI() ffi_libc.cdef(""" int vasprintf(char **strp, const char *fmt, void *ap); diff --git a/openpilot/system/ui/lib/wifi_manager.py b/openpilot/system/ui/lib/wifi_manager.py index 26474be942c1d0..221022fb8ba03e 100644 --- a/openpilot/system/ui/lib/wifi_manager.py +++ b/openpilot/system/ui/lib/wifi_manager.py @@ -166,7 +166,7 @@ def __init__(self): _wrap_router(self._router_main) self._conn_monitor = open_dbus_connection_blocking(bus="SYSTEM") # used by state monitor thread self._nm = DBusAddress(NM_PATH, bus_name=NM, interface=NM_IFACE) - except FileNotFoundError: + except (FileNotFoundError, AttributeError): # AttributeError: no AF_UNIX sockets on Windows cloudlog.exception("Failed to connect to system D-Bus") self._router_main = None self._conn_monitor = None @@ -203,7 +203,8 @@ def __init__(self): self._scan_lock = threading.Lock() self._scan_thread = threading.Thread(target=self._network_scanner, daemon=True) self._state_thread = threading.Thread(target=self._monitor_state, daemon=True) - self._initialize() + if not self._exit: # no D-Bus, nothing to scan with + self._initialize() atexit.register(self.stop) def _initialize(self): diff --git a/openpilot/system/webrtc/tests/test_stream_session.py b/openpilot/system/webrtc/tests/test_stream_session.py index 03d540c3c2d630..c121891c47b6e1 100644 --- a/openpilot/system/webrtc/tests/test_stream_session.py +++ b/openpilot/system/webrtc/tests/test_stream_session.py @@ -1,11 +1,16 @@ import asyncio import json import time +import unittest import capnp from openpilot.common.test import OpenpilotTestCase from openpilot.cereal import messaging, log -from teleoprtc.tracks import VIDEO_CLOCK_RATE + +try: + from teleoprtc.tracks import VIDEO_CLOCK_RATE +except ImportError as e: # TODO: drop when teleoprtc installs on Windows (libdatachannel-py wheels) + raise unittest.SkipTest(f"teleoprtc unavailable: {e}") from None from openpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy, ServerState, handle_get_stream from openpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack diff --git a/openpilot/system/webrtc/webrtcd.py b/openpilot/system/webrtc/webrtcd.py index 9481e077abdb5c..7dc0cd418eceff 100755 --- a/openpilot/system/webrtc/webrtcd.py +++ b/openpilot/system/webrtc/webrtcd.py @@ -626,7 +626,10 @@ def request_shutdown() -> None: shutdown_task = loop.create_task(_shutdown(server, state, loop)) for sig in (signal.SIGINT, signal.SIGTERM): - loop.add_signal_handler(sig, request_shutdown) + try: + loop.add_signal_handler(sig, request_shutdown) + except NotImplementedError: # Windows event loops have no signal handlers + signal.signal(sig, lambda *_: loop.call_soon_threadsafe(request_shutdown)) try: loop.run_forever() diff --git a/openpilot/test_native.py b/openpilot/test_native.py index eed549f4e47b4b..51a35805952bc2 100644 --- a/openpilot/test_native.py +++ b/openpilot/test_native.py @@ -1,5 +1,6 @@ import os import subprocess +import sysconfig from openpilot.common.basedir import BASEDIR from openpilot.common.parameterized import parameterized @@ -16,7 +17,7 @@ class TestNative(OpenpilotTestCase): @parameterized.expand(NATIVE_TESTS) def test_native(self, executable): - path = os.path.join(BASEDIR, executable) + path = os.path.join(BASEDIR, executable) + sysconfig.get_config_var("EXE") # .exe on Windows if not os.path.exists(path): self.skipTest(f"optional native test was not built: {executable}") subprocess.run([path], check=True) diff --git a/openpilot/tools/cabana/SConscript b/openpilot/tools/cabana/SConscript index 0249a019bbaddd..927549f42448ae 100644 --- a/openpilot/tools/cabana/SConscript +++ b/openpilot/tools/cabana/SConscript @@ -7,7 +7,7 @@ from openpilot.common.basedir import BASEDIR Import('env', 'arch', 'common', 'messaging', 'visionipc', 'cereal', 'replay_lib', 'ffmpeg_libs') -opendbc_path = '-DOPENDBC_FILE_PATH=\'"%s"\'' % (env.Dir("../../../opendbc_repo/opendbc/dbc").abspath) +opendbc_path = '-DOPENDBC_FILE_PATH=\'"%s"\'' % (env.Dir("../../../opendbc_repo/opendbc/dbc").abspath.replace(os.sep, "/")) # embed the bootstrap icons SVG into the binary def build_bootstrap_icons_src(target, source, env): @@ -29,7 +29,7 @@ bootstrap_icons_src = env.Command('assets/bootstrap_icons.cc', str(bootstrap_ico core_srcs = ['streams/pandastream.cc', 'streams/devicestream.cc', 'streams/livestream.cc', 'streams/abstractstream.cc', 'streams/replaystream.cc', 'dbc/dbc.cc', 'dbc/dbcfile.cc', 'dbc/dbcmanager.cc', 'utils/export.cc', 'utils/util.cc', 'utils/strings.cc', 'commands.cc', 'settings.cc', 'routes.cc', 'panda.cc'] -if arch != "Darwin": +if arch not in ("Darwin", "Windows"): core_srcs += ['streams/socketcanstream.cc'] # imgui frontend (tools/cabana/ui), no Qt @@ -39,8 +39,8 @@ ui_env['LIBPATH'] += [imgui.MESA_DIR, libusb.LIB_DIR] ui_env['CXXFLAGS'] += [ opendbc_path, "-DGLFW_INCLUDE_NONE", - '-DCABANA_FONTS_DIR=\'"%s"\'' % os.path.join(os.path.realpath(BASEDIR), "openpilot", "selfdrive", "assets", "fonts"), - '-DBOOTSTRAP_ICONS_TTF=\'"%s"\'' % bootstrap_icons.TTF_PATH, + '-DCABANA_FONTS_DIR=\'"%s"\'' % os.path.join(os.path.realpath(BASEDIR), "openpilot", "selfdrive", "assets", "fonts").replace(os.sep, "/"), + '-DBOOTSTRAP_ICONS_TTF=\'"%s"\'' % bootstrap_icons.TTF_PATH.as_posix(), ] ui_objs = [ui_env.Object('ui/obj/' + src.replace('/', '_')[:-3], src) for src in core_srcs] ui_objs += [ui_env.Object('ui/obj/bootstrap_icons', bootstrap_icons_src)] @@ -49,6 +49,8 @@ ui_libs = [replay_lib, common, messaging, visionipc, cereal, File(f"{imgui.LIB_D ffmpeg_libs + ['zstd', 'm', 'pthread', 'usb-1.0'] if arch == "Darwin": ui_env['FRAMEWORKS'] = ['OpenGL', 'Cocoa', 'IOKit', 'CoreFoundation', 'CoreVideo', 'CoreMedia', 'Security', 'VideoToolbox'] +elif arch == "Windows": + ui_libs += ['opengl32', 'gdi32', 'winmm', 'shell32', 'user32', 'setupapi', 'dwmapi', 'dl'] else: ui_libs += ['GL', 'dl'] cabana_ui = ui_env.Program('cabana', ui_objs, LIBS=ui_libs) diff --git a/openpilot/tools/cabana/panda.cc b/openpilot/tools/cabana/panda.cc index 9bb5320eaca485..0f826b59871f69 100644 --- a/openpilot/tools/cabana/panda.cc +++ b/openpilot/tools/cabana/panda.cc @@ -1,5 +1,10 @@ #include "tools/cabana/panda.h" +#ifdef _WIN32 +#include "common/win32.h" +#endif +#include + #include #include #include diff --git a/openpilot/tools/cabana/panda.h b/openpilot/tools/cabana/panda.h index c99be1f5cfd953..e2621335ba00a6 100644 --- a/openpilot/tools/cabana/panda.h +++ b/openpilot/tools/cabana/panda.h @@ -11,7 +11,10 @@ #include #include -#include + +// libusb.h pulls in on Windows, which clashes with the capnp and params enums, so only panda.cc includes it +struct libusb_context; +struct libusb_device_handle; #include "openpilot/cereal/gen/cpp/car.capnp.h" #include "openpilot/cereal/gen/cpp/log.capnp.h" diff --git a/openpilot/tools/cabana/routes.cc b/openpilot/tools/cabana/routes.cc index 50a1eacf8d8e79..17fb4ec5ff299d 100644 --- a/openpilot/tools/cabana/routes.cc +++ b/openpilot/tools/cabana/routes.cc @@ -1,5 +1,7 @@ #include "tools/cabana/routes.h" +#include "common/util.h" + #include #include #include diff --git a/openpilot/tools/cabana/settings.cc b/openpilot/tools/cabana/settings.cc index 7f8e6473c2b9a1..708124e2fa4a89 100644 --- a/openpilot/tools/cabana/settings.cc +++ b/openpilot/tools/cabana/settings.cc @@ -17,8 +17,10 @@ #include #include -#include #include +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif #ifdef __APPLE__ #include @@ -26,6 +28,7 @@ #include +#include "common/file.h" #include "json11/json11.hpp" #include "tools/cabana/utils/util.h" @@ -46,9 +49,9 @@ struct LoadedSettings { class FileLock { public: explicit FileLock(const std::filesystem::path &path) { - fd = open(path.c_str(), O_CREAT | O_CLOEXEC, 0600); - if (fd < 0 || flock(fd, LOCK_EX) < 0) { - fprintf(stderr, "failed to lock Cabana settings %s: %s\n", path.c_str(), strerror(errno)); + fd = open(path.string().c_str(), O_CREAT | O_CLOEXEC, 0600); + if (fd < 0 || util::lock_file_exclusive(fd) < 0) { + fprintf(stderr, "failed to lock Cabana settings %s: %s\n", path.string().c_str(), strerror(errno)); if (fd >= 0) close(fd); fd = -1; } @@ -70,7 +73,7 @@ LoadedSettings loadSettings() { std::string error; auto settings_json = json11::Json::parse(contents, error); if (!error.empty() || !settings_json.is_object()) { - fprintf(stderr, "failed to read Cabana settings %s%s%s\n", settingsFile().c_str(), error.empty() ? "" : ": ", error.c_str()); + fprintf(stderr, "failed to read Cabana settings %s%s%s\n", settingsFile().string().c_str(), error.empty() ? "" : ": ", error.c_str()); return {.exists = true, .valid = false}; } return {.values = settings_json.object_items(), .exists = true}; @@ -81,7 +84,7 @@ bool ensureSettingsDirectory() { std::error_code error; std::filesystem::create_directories(path.parent_path(), error); if (error) { - fprintf(stderr, "failed to create Cabana settings directory %s: %s\n", path.parent_path().c_str(), error.message().c_str()); + fprintf(stderr, "failed to create Cabana settings directory %s: %s\n", path.parent_path().string().c_str(), error.message().c_str()); return false; } return true; @@ -110,18 +113,20 @@ bool saveSettings(const json11::Json::object &settings_json) { bool success = writeAll(fd, contents) && fsync(fd) == 0; if (close(fd) < 0) success = false; - if (success && rename(temporary_path.c_str(), path.c_str()) < 0) success = false; + if (success && util::replace_file(temporary_path.c_str(), path.string().c_str()) < 0) success = false; +#ifndef _WIN32 // directories cannot be fsynced through the Windows CRT if (success) { int dir_fd = open(path.parent_path().c_str(), O_RDONLY | O_CLOEXEC); success = dir_fd >= 0 && fsync(dir_fd) == 0; if (dir_fd >= 0 && close(dir_fd) < 0) success = false; } +#endif if (!success) { const int saved_errno = errno; unlink(temporary_path.c_str()); - fprintf(stderr, "failed to save Cabana settings to %s: %s\n", path.c_str(), strerror(saved_errno)); + fprintf(stderr, "failed to save Cabana settings to %s: %s\n", path.string().c_str(), strerror(saved_errno)); } return success; } @@ -134,11 +139,11 @@ bool preserveCorruptSettings() { backup = path; backup += ".corrupt." + std::to_string(i); } - if (rename(path.c_str(), backup.c_str()) < 0) { - fprintf(stderr, "failed to preserve corrupt Cabana settings %s: %s\n", path.c_str(), strerror(errno)); + if (util::replace_file(path.string().c_str(), backup.string().c_str()) < 0) { + fprintf(stderr, "failed to preserve corrupt Cabana settings %s: %s\n", path.string().c_str(), strerror(errno)); return false; } - fprintf(stderr, "preserved corrupt Cabana settings at %s\n", backup.c_str()); + fprintf(stderr, "preserved corrupt Cabana settings at %s\n", backup.string().c_str()); return true; } diff --git a/openpilot/tools/cabana/streams/devicestream.cc b/openpilot/tools/cabana/streams/devicestream.cc index 986ca13558c688..c9dfc99c0e914a 100644 --- a/openpilot/tools/cabana/streams/devicestream.cc +++ b/openpilot/tools/cabana/streams/devicestream.cc @@ -12,8 +12,13 @@ #include #include #include +#ifdef _WIN32 +#include "common/win32.h" +#else #include +#endif +#include "common/util.h" #include "openpilot/cereal/services.h" #include "tools/cabana/utils/util.h" @@ -27,6 +32,36 @@ DeviceStream::~DeviceStream() { stopBridge(); } +#ifdef _WIN32 +void DeviceStream::stopBridge() { + if (bridge_process == nullptr) return; + TerminateProcess(bridge_process, 0); + WaitForSingleObject(bridge_process, 3000); + CloseHandle(bridge_process); + bridge_process = nullptr; +} + +void DeviceStream::start() { + if (!zmq_address.empty()) { + stopBridge(); + const std::string path = (executableDir() / "../../cereal/messaging/bridge.exe").lexically_normal().string(); + // CreateProcess re-parses the command line, so the can filter argument needs its quotes escaped + std::string cmdline = "\"" + path + "\" " + zmq_address + " \"/\\\"can/\\\"\""; + + STARTUPINFOA si = {}; + si.cb = sizeof(si); + PROCESS_INFORMATION pi = {}; + if (!CreateProcessA(path.c_str(), cmdline.data(), NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, NULL, &si, &pi)) { + error("Failed to start bridge: error " + std::to_string(GetLastError())); + return; + } + CloseHandle(pi.hThread); + bridge_process = pi.hProcess; + } + + LiveStream::start(); +} +#else void DeviceStream::stopBridge() { if (bridge_pid <= 0) return; @@ -92,6 +127,7 @@ void DeviceStream::start() { LiveStream::start(); } +#endif void DeviceStream::streamThread() { zmq_address.empty() ? unsetenv("ZMQ") : setenv("ZMQ", "1", 1); diff --git a/openpilot/tools/cabana/streams/devicestream.h b/openpilot/tools/cabana/streams/devicestream.h index 3770d952aa5c9c..953bfb09a85d5a 100644 --- a/openpilot/tools/cabana/streams/devicestream.h +++ b/openpilot/tools/cabana/streams/devicestream.h @@ -17,6 +17,10 @@ class DeviceStream : public LiveStream { void start() override; void streamThread() override; void stopBridge(); +#ifdef _WIN32 + void *bridge_process = nullptr; // HANDLE +#else pid_t bridge_pid = -1; +#endif const std::string zmq_address; }; diff --git a/openpilot/tools/cabana/streams/pandastream.cc b/openpilot/tools/cabana/streams/pandastream.cc index 0e72443c7afc2b..6f33e2b19c74f4 100644 --- a/openpilot/tools/cabana/streams/pandastream.cc +++ b/openpilot/tools/cabana/streams/pandastream.cc @@ -60,7 +60,7 @@ void PandaStream::streamThread() { MessageBuilder msg; auto evt = msg.initEvent(); auto canData = evt.initCan(raw_can_data.size()); - for (uint i = 0; i +#include "common/hardware/hw.h" #include "common/timing.h" #include "common/util.h" #include "tools/cabana/settings.h" ReplayStream::ReplayStream() { unsetenv("ZMQ"); - setenv("COMMA_CACHE", "/tmp/comma_download_cache", 1); + setenv("COMMA_CACHE", (Path::tmp_dir() + "/comma_download_cache").c_str(), 1); op_prefix = std::make_unique(); diff --git a/openpilot/tools/cabana/tests/test_cabana.cc b/openpilot/tools/cabana/tests/test_cabana.cc index 268aa9a86d7fcd..ad518dc61692a8 100644 --- a/openpilot/tools/cabana/tests/test_cabana.cc +++ b/openpilot/tools/cabana/tests/test_cabana.cc @@ -10,6 +10,7 @@ #include #include "common/tests/native_test.h" +#include "common/util.h" #include "tools/cabana/dbc/dbcfile.h" #include "tools/cabana/dbc/dbcmanager.h" #include "tools/cabana/routes.h" diff --git a/openpilot/tools/cabana/tests/test_cabana_ui.py b/openpilot/tools/cabana/tests/test_cabana_ui.py index aecd7e6ed7f369..fab6eca7062bb1 100644 --- a/openpilot/tools/cabana/tests/test_cabana_ui.py +++ b/openpilot/tools/cabana/tests/test_cabana_ui.py @@ -1,3 +1,4 @@ +import os import subprocess from pathlib import Path @@ -8,6 +9,6 @@ class TestCabanaUi(OpenpilotTestCase): def test_help(self): - result = subprocess.run(["./cabana", "-h"], cwd=CABANA_DIR, capture_output=True, text=True) + result = subprocess.run([os.path.join(CABANA_DIR, "cabana"), "-h"], cwd=CABANA_DIR, capture_output=True, text=True) assert result.returncode == 0, result.stderr assert "Usage:" in result.stderr diff --git a/openpilot/tools/cabana/ui/app.cc b/openpilot/tools/cabana/ui/app.cc index 3f4dac101671ff..e5b5aa673eaaf2 100644 --- a/openpilot/tools/cabana/ui/app.cc +++ b/openpilot/tools/cabana/ui/app.cc @@ -1,5 +1,9 @@ #include "tools/cabana/ui/app.h" +#ifdef _WIN32 +#include "common/win32.h" // before the imgui OpenGL loader pulls in with its clashing macros +#endif + #include #include #include @@ -13,6 +17,13 @@ #include "imgui_impl_opengl3_loader.h" #include "implot.h" #include +#ifdef _WIN32 +#define GLFW_EXPOSE_NATIVE_WIN32 +#include +// dwmapi.h needs the GDI types our lean windows.h leaves out; this is the only thing used from it +extern "C" HRESULT WINAPI DwmSetWindowAttribute(HWND hwnd, DWORD attribute, LPCVOID value, DWORD size); +constexpr DWORD DWMWA_USE_IMMERSIVE_DARK_MODE = 20; +#endif #include "tools/cabana/settings.h" #include "tools/cabana/ui/inistate.h" @@ -200,6 +211,15 @@ std::vector takeKeyEvents() { return std::exchange(g_key_events, {}); } +void applyTitleBarTheme(bool dark) { +#ifdef _WIN32 + if (GLFWwindow *window = glfwGetCurrentContext()) { + BOOL value = dark; + DwmSetWindowAttribute(glfwGetWin32Window(window), DWMWA_USE_IMMERSIVE_DARK_MODE, &value, sizeof(value)); + } +#endif +} + int run(std::unique_ptr stream, StreamLoader stream_loader, const std::string &dbc_file) { try { // SIGINT/SIGTERM close all windows (which may ask about unsaved changes), then exit diff --git a/openpilot/tools/cabana/ui/dialogs/streamselector.cc b/openpilot/tools/cabana/ui/dialogs/streamselector.cc index f1d2b1da136558..469f0fe9d8d51e 100644 --- a/openpilot/tools/cabana/ui/dialogs/streamselector.cc +++ b/openpilot/tools/cabana/ui/dialogs/streamselector.cc @@ -1,5 +1,7 @@ #include "tools/cabana/ui/dialogs/streamselector.h" +#include "common/util.h" + #include #include #include diff --git a/openpilot/tools/cabana/ui/theme.cc b/openpilot/tools/cabana/ui/theme.cc index ff02ca24c7c706..db5e3015efce8a 100644 --- a/openpilot/tools/cabana/ui/theme.cc +++ b/openpilot/tools/cabana/ui/theme.cc @@ -59,7 +59,7 @@ ImFont *addFont(const fs::path &path, float size) { ImFontConfig cfg; cfg.OversampleH = 2; cfg.OversampleV = 2; - ImFont *font = ImGui::GetIO().Fonts->AddFontFromFileTTF(path.c_str(), size, &cfg); + ImFont *font = ImGui::GetIO().Fonts->AddFontFromFileTTF(path.string().c_str(), size, &cfg); if (font != nullptr) addIconFont(size, font); return font; } @@ -83,6 +83,7 @@ void loadFonts() { void applyTheme(int theme) { g_dark = theme == DARK_THEME; + applyTitleBarTheme(g_dark); g_palette = g_dark ? &DARK_PALETTE : &LIGHT_PALETTE; const Palette &p = *g_palette; const ImVec4 none(0, 0, 0, 0); diff --git a/openpilot/tools/cabana/ui/theme.h b/openpilot/tools/cabana/ui/theme.h index 91f2a0740ff8d6..8042ad9c3e4ccc 100644 --- a/openpilot/tools/cabana/ui/theme.h +++ b/openpilot/tools/cabana/ui/theme.h @@ -36,6 +36,7 @@ constexpr float UI_FONT_SIZE = 16.0f; void loadFonts(); void applyTheme(int theme); // Safe to call at runtime. bool isDarkTheme(); +void applyTitleBarTheme(bool dark); // the OS-drawn title bar, where the OS allows it (Windows) const Palette &palette(); CabanaColor signalFillColor(const CabanaColor &c); diff --git a/openpilot/tools/cabana/utils/strings.cc b/openpilot/tools/cabana/utils/strings.cc index 3a1191609e9f0f..518d16e1281d59 100644 --- a/openpilot/tools/cabana/utils/strings.cc +++ b/openpilot/tools/cabana/utils/strings.cc @@ -5,6 +5,10 @@ #include #include +#ifdef _WIN32 +static void localtime_r(const std::time_t *t, std::tm *out) { localtime_s(out, t); } +#endif + #include "tools/cabana/dbc/dbc.h" namespace utils { diff --git a/openpilot/tools/cabana/utils/util.cc b/openpilot/tools/cabana/utils/util.cc index 1d1c8c75e4c164..c4b75c6e116b7a 100644 --- a/openpilot/tools/cabana/utils/util.cc +++ b/openpilot/tools/cabana/utils/util.cc @@ -14,9 +14,13 @@ #include #include #include +#ifdef _WIN32 +#include "common/win32.h" +#else #include #include #include +#endif #ifdef __APPLE__ #include #endif @@ -81,6 +85,33 @@ std::pair SegmentTree::get_minmax(int n, int left, int right, in // UnixSignalHandler +#ifdef _WIN32 +UnixSignalHandler::UnixSignalHandler(std::function on_signal) { + sig_event = CreateEventA(NULL, TRUE, FALSE, NULL); + + waiter = std::thread([this, on_signal = std::move(on_signal)]() { + WaitForSingleObject(sig_event, INFINITE); + if (shutting_down.load()) return; + + on_signal(); + }); + + std::signal(SIGINT, signalHandler); + std::signal(SIGTERM, UnixSignalHandler::signalHandler); +} + +UnixSignalHandler::~UnixSignalHandler() { + shutting_down.store(true); + SetEvent(sig_event); + if (waiter.joinable()) waiter.join(); + CloseHandle(sig_event); +} + +void UnixSignalHandler::signalHandler(int s) { + (void)s; + SetEvent(sig_event); +} +#else UnixSignalHandler::UnixSignalHandler(std::function on_signal) { if (::socketpair(AF_UNIX, SOCK_STREAM, 0, sig_fd)) { fprintf(stderr, "Couldn't create TERM socketpair\n"); @@ -113,6 +144,7 @@ UnixSignalHandler::~UnixSignalHandler() { void UnixSignalHandler::signalHandler(int s) { (void)!::write(sig_fd[0], &s, sizeof(s)); } +#endif // validators @@ -243,13 +275,20 @@ static std::unordered_map load_bootstrap_icons() { namespace utils { std::string homePath() { +#ifdef _WIN32 + const char *home = ::getenv("USERPROFILE"); +#else const char *home = ::getenv("HOME"); +#endif return home ? home : ""; } std::filesystem::path configPath() { #ifdef __APPLE__ return std::filesystem::path(homePath()) / "Library/Preferences"; +#elif defined(_WIN32) + const char *appdata = ::getenv("APPDATA"); + return (appdata && appdata[0]) ? std::filesystem::path(appdata) : std::filesystem::path(homePath()) / ".config"; #else const char *xdg = ::getenv("XDG_CONFIG_HOME"); return (xdg && xdg[0]) ? std::filesystem::path(xdg) : std::filesystem::path(homePath()) / ".config"; @@ -259,6 +298,9 @@ std::filesystem::path configPath() { #ifdef __APPLE__ static const char *clipboard_read_cmds[] = {"pbpaste"}; static const char *clipboard_write_cmds[] = {"pbcopy"}; +#elif defined(_WIN32) +static const char *clipboard_read_cmds[] = {"powershell -NoProfile -Command Get-Clipboard -Raw"}; +static const char *clipboard_write_cmds[] = {"clip"}; #else static const char *clipboard_read_cmds[] = {"wl-paste --no-newline 2>/dev/null", "xclip -selection clipboard -o 2>/dev/null", "xsel -ob 2>/dev/null"}; static const char *clipboard_write_cmds[] = {"wl-copy 2>/dev/null", "xclip -selection clipboard 2>/dev/null", "xsel -ib 2>/dev/null"}; @@ -284,7 +326,9 @@ bool getClipboardText(std::string *text) { } bool setClipboardText(const std::string &text) { +#ifndef _WIN32 std::signal(SIGPIPE, SIG_IGN); +#endif for (const char *cmd : clipboard_write_cmds) { FILE *f = ::popen(cmd, "w"); if (!f) continue; @@ -317,6 +361,10 @@ std::filesystem::path executableDir() { std::error_code ec; auto path = std::filesystem::canonical(buf, ec); return (ec ? std::filesystem::path(buf) : path).parent_path(); +#elif defined(_WIN32) + char buf[MAX_PATH] = {}; + GetModuleFileNameA(NULL, buf, sizeof(buf)); + return std::filesystem::path(buf).parent_path(); #else return std::filesystem::path(util::readlink("/proc/self/exe")).parent_path(); #endif diff --git a/openpilot/tools/cabana/utils/util.h b/openpilot/tools/cabana/utils/util.h index 3570320f346c83..f0f08c9c38d2d6 100644 --- a/openpilot/tools/cabana/utils/util.h +++ b/openpilot/tools/cabana/utils/util.h @@ -95,7 +95,11 @@ class UnixSignalHandler { static void signalHandler(int s); private: +#ifdef _WIN32 + inline static void *sig_event = nullptr; // HANDLE; the console control handler has no fds to write to +#else inline static int sig_fd[2] = {}; +#endif std::atomic shutting_down{false}; std::thread waiter; }; diff --git a/openpilot/tools/jotpluggler/SConscript b/openpilot/tools/jotpluggler/SConscript index bfe6e907427a6f..c1e62e6e4b713b 100644 --- a/openpilot/tools/jotpluggler/SConscript +++ b/openpilot/tools/jotpluggler/SConscript @@ -16,8 +16,8 @@ jot_env["LIBPATH"] += [imgui.MESA_DIR, libusb.LIB_DIR] jot_env["CPPPATH"] += [imgui.INCLUDE_DIR, libusb.INCLUDE_DIR] jot_env["CXXFLAGS"] += [ "-DGLFW_INCLUDE_NONE", - '-DJOTP_REPO_ROOT=\'"%s"\'' % os.path.realpath(BASEDIR), - '-DBOOTSTRAP_ICONS_TTF=\'"%s"\'' % bootstrap_icons.TTF_PATH, + '-DJOTP_REPO_ROOT=\'"%s"\'' % os.path.realpath(BASEDIR).replace(os.sep, "/"), + '-DBOOTSTRAP_ICONS_TTF=\'"%s"\'' % bootstrap_icons.TTF_PATH.as_posix(), ] def materialize_generated_dbcs(target, source, env): @@ -29,10 +29,10 @@ def materialize_generated_dbcs(target, source, env): os.unlink(os.path.join(out_dir, name)) for name, content in sorted(get_generated_dbcs().items()): - with open(os.path.join(out_dir, f"{name}.dbc"), "w") as f: + with open(os.path.join(out_dir, f"{name}.dbc"), "w", encoding="utf-8") as f: f.write(content) - with open(str(target[0]), "w") as f: + with open(str(target[0]), "w", encoding="utf-8") as f: f.write("ok\n") return None @@ -75,7 +75,7 @@ def write_car_fingerprint_to_dbc_header(target, source, env): "", ]) - with open(str(target[0]), "w") as f: + with open(str(target[0]), "w", encoding="utf-8") as f: f.write("\n".join(lines)) return None @@ -105,6 +105,8 @@ libs = [replay_lib, common, messaging, visionipc, cereal, File(f"{imgui.LIB_DIR} ffmpeg_libs + ["zstd", "m", "pthread", "usb-1.0"] if arch == "Darwin": jot_env["FRAMEWORKS"] = ["OpenGL", "Cocoa", "IOKit", "CoreFoundation", "CoreVideo", "CoreMedia", "VideoToolbox"] +elif arch == "Windows": + libs += ["opengl32", "gdi32", "winmm", "shell32", "user32", "setupapi", "dl"] else: libs += ["GL", "dl"] diff --git a/openpilot/tools/jotpluggler/app.cc b/openpilot/tools/jotpluggler/app.cc index e6ba696bae8c95..15d2cbf6890502 100644 --- a/openpilot/tools/jotpluggler/app.cc +++ b/openpilot/tools/jotpluggler/app.cc @@ -257,7 +257,7 @@ void configure_style() { font_cfg.RasterizerDensity = 1.0f; icon_add_font(16.0f); const auto add_font_with_icons = [&](const fs::path &path, float size) -> ImFont * { - ImFont *font = io.Fonts->AddFontFromFileTTF(path.c_str(), size, &font_cfg); + ImFont *font = io.Fonts->AddFontFromFileTTF(path.string().c_str(), size, &font_cfg); if (font != nullptr) { icon_add_font(size, true, font); } diff --git a/openpilot/tools/jotpluggler/common.cc b/openpilot/tools/jotpluggler/common.cc index 50f5fc0b95810f..43983f60604528 100644 --- a/openpilot/tools/jotpluggler/common.cc +++ b/openpilot/tools/jotpluggler/common.cc @@ -3,6 +3,10 @@ #include #include #include +#ifdef _WIN32 +#include "common/win32.h" +#include +#endif namespace { @@ -148,12 +152,16 @@ bool app_begin_popup_modal(const char *name, bool *p_open, ImGuiWindowFlags flag } void open_external_url(std::string_view url) { +#ifdef _WIN32 + ShellExecuteA(NULL, "open", std::string(url).c_str(), NULL, NULL, SW_SHOWNORMAL); +#else #ifdef __APPLE__ const std::string command = "open " + shell_quote(url) + " &"; #else const std::string command = "xdg-open " + shell_quote(url) + " >/dev/null 2>&1 &"; #endif util::check_system(command); +#endif } std::string route_useradmin_url(const RouteIdentifier &route_id) { diff --git a/openpilot/tools/jotpluggler/generate_event_extractors.py b/openpilot/tools/jotpluggler/generate_event_extractors.py index a424ebc237b694..9f9c0d111381d2 100644 --- a/openpilot/tools/jotpluggler/generate_event_extractors.py +++ b/openpilot/tools/jotpluggler/generate_event_extractors.py @@ -183,7 +183,7 @@ def emit_list(self, indent, type_proto, schema, list_expr, path, path_expr, dyna if elem_scalar is not None: self.emit(indent, f"if ({list_expr}.size() <= 16) {{") index_var = self.tmp("i") - self.emit(indent + 2, f"for (uint {index_var} = 0; {index_var} < {list_expr}.size(); ++{index_var}) {{") + self.emit(indent + 2, f"for (unsigned int {index_var} = 0; {index_var} < {list_expr}.size(); ++{index_var}) {{") item_series = self.tmp("item_series") self.emit(indent + 4, f"RouteSeries *{item_series} = ensure_list_scalar_series({base_path_var}, {index_var}, series);") if elem_scalar == "Enum": @@ -195,7 +195,7 @@ def emit_list(self, indent, type_proto, schema, list_expr, path, path_expr, dyna if elem_kind in {"struct", "list"}: index_var = self.tmp("i") - self.emit(indent, f"for (uint {index_var} = 0; {index_var} < {list_expr}.size(); ++{index_var}) {{") + self.emit(indent, f"for (unsigned int {index_var} = 0; {index_var} < {list_expr}.size(); ++{index_var}) {{") item_path = self.tmp("item_path") self.emit(indent + 2, f"const std::string {item_path} = {base_path_var} + \"/\" + std::to_string({index_var});") item = self.tmp("item") diff --git a/openpilot/tools/jotpluggler/icons.cc b/openpilot/tools/jotpluggler/icons.cc index 29edabad4ee3df..55b3a83b9ee793 100644 --- a/openpilot/tools/jotpluggler/icons.cc +++ b/openpilot/tools/jotpluggler/icons.cc @@ -15,7 +15,7 @@ void icon_add_font(float size, bool merge, const ImFont *base_font) { config.GlyphOffset.y = std::round(size * 0.5f - base_center); } static const ImWchar ranges[] = {0xF000, 0xF8FF, 0}; - io.Fonts->AddFontFromFileTTF(ttf.c_str(), size, &config, ranges); + io.Fonts->AddFontFromFileTTF(ttf.string().c_str(), size, &config, ranges); } bool icon_menu_item(const char *glyph, const char *label, const char *shortcut, bool selected, bool enabled) { diff --git a/openpilot/tools/jotpluggler/map.cc b/openpilot/tools/jotpluggler/map.cc index fb4e03c9122a34..fb7af15c06f905 100644 --- a/openpilot/tools/jotpluggler/map.cc +++ b/openpilot/tools/jotpluggler/map.cc @@ -425,7 +425,11 @@ uint64_t fnv1a64(std::string_view text) { } fs::path basemap_cache_root() { +#ifdef _WIN32 + const char *home = std::getenv("USERPROFILE"); +#else const char *home = std::getenv("HOME"); +#endif fs::path root = home != nullptr ? fs::path(home) / ".comma" : fs::temp_directory_path(); root /= "jotpluggler_vector_map"; fs::create_directories(root); diff --git a/openpilot/tools/jotpluggler/test_jotpluggler.py b/openpilot/tools/jotpluggler/test_jotpluggler.py index cbcc0a81687316..7d079390a87660 100644 --- a/openpilot/tools/jotpluggler/test_jotpluggler.py +++ b/openpilot/tools/jotpluggler/test_jotpluggler.py @@ -1,3 +1,4 @@ +import os import subprocess from pathlib import Path @@ -8,6 +9,6 @@ from openpilot.common.test import OpenpilotTestCase class TestJotpluggler(OpenpilotTestCase): def test_help(self): - result = subprocess.run(["./jotpluggler", "-h"], cwd=JOTPLUGGLER_DIR, capture_output=True, text=True) + result = subprocess.run([os.path.join(JOTPLUGGLER_DIR, "jotpluggler"), "-h"], cwd=JOTPLUGGLER_DIR, capture_output=True, text=True) assert result.returncode == 0, result.stderr assert "Usage:" in result.stderr diff --git a/openpilot/tools/jotpluggler/util.cc b/openpilot/tools/jotpluggler/util.cc index 5c20e795f6ba06..0f4f84dec379ae 100644 --- a/openpilot/tools/jotpluggler/util.cc +++ b/openpilot/tools/jotpluggler/util.cc @@ -3,7 +3,9 @@ #include #include #include +#ifndef _WIN32 #include +#endif std::string read_file_or_throw(const std::filesystem::path &path) { const std::string contents = util::read_file(path.string()); diff --git a/openpilot/tools/lib/tests/test_logreader.py b/openpilot/tools/lib/tests/test_logreader.py index 9d5691abd881cd..3c78ab467f8ad6 100644 --- a/openpilot/tools/lib/tests/test_logreader.py +++ b/openpilot/tools/lib/tests/test_logreader.py @@ -259,7 +259,8 @@ def test_sort_by_time(self): assert msgs == sorted(msgs, key=lambda m: m.logMonoTime) def test_only_union_types(self): - with tempfile.NamedTemporaryFile() as qlog: + with tempfile.NamedTemporaryFile(delete_on_close=False) as qlog: + qlog.close() # Windows: the file cannot be reopened while this handle is open # write valid Event messages num_msgs = 100 with open(qlog.name, "wb") as f: diff --git a/openpilot/tools/lib/url_file.py b/openpilot/tools/lib/url_file.py index ec0a3d58153c7b..6b80f294651489 100644 --- a/openpilot/tools/lib/url_file.py +++ b/openpilot/tools/lib/url_file.py @@ -217,4 +217,5 @@ def name(self) -> str: return self._url -os.register_at_fork(after_in_child=URLFile.reset) +if hasattr(os, "register_at_fork"): # no fork on Windows + os.register_at_fork(after_in_child=URLFile.reset) diff --git a/openpilot/tools/replay/SConscript b/openpilot/tools/replay/SConscript index 9e060bc82946fb..dcd85188f94511 100644 --- a/openpilot/tools/replay/SConscript +++ b/openpilot/tools/replay/SConscript @@ -8,7 +8,7 @@ base_libs = [common, messaging, cereal, visionipc, 'm', 'pthread'] replay_lib_src = ["replay.cc", "consoleui.cc", "camera.cc", "filereader.cc", "logreader.cc", "framereader.cc", "route.cc", "util.cc", "seg_mgr.cc", "timeline.cc", "py_downloader.cc"] -if arch != "Darwin": +if arch not in ("Darwin", "Windows"): replay_lib_src.append("#openpilot/system/loggerd/encoder/v4l_decoder.cc") replay_lib = replay_env.Library("replay", replay_lib_src, LIBS=base_libs, FRAMEWORKS=base_frameworks) Export('replay_lib') diff --git a/openpilot/tools/replay/consoleui.cc b/openpilot/tools/replay/consoleui.cc index aa58cc959d6b04..e2de718777b9cf 100644 --- a/openpilot/tools/replay/consoleui.cc +++ b/openpilot/tools/replay/consoleui.cc @@ -72,14 +72,15 @@ ConsoleUI::ConsoleUI(Replay *replay) : replay(replay), sm({"carState", "vehicleP // Initialize all the colors. https://www.ditig.com/256-colors-cheat-sheet start_color(); - init_pair(Color::Debug, 246, COLOR_BLACK); // #949494 - init_pair(Color::Yellow, 184, COLOR_BLACK); + auto color = [](int xterm256, int basic) { return COLORS >= 256 ? xterm256 : basic; }; // 8/16-color terminals + init_pair(Color::Debug, color(246, COLOR_WHITE), COLOR_BLACK); // #949494 + init_pair(Color::Yellow, color(184, COLOR_YELLOW), COLOR_BLACK); init_pair(Color::Red, COLOR_RED, COLOR_BLACK); init_pair(Color::Cyan, COLOR_CYAN, COLOR_BLACK); - init_pair(Color::BrightWhite, 15, COLOR_BLACK); + init_pair(Color::BrightWhite, color(15, COLOR_WHITE), COLOR_BLACK); init_pair(Color::Disengaged, COLOR_BLUE, COLOR_BLUE); - init_pair(Color::Engaged, 28, 28); - init_pair(Color::Green, 34, COLOR_BLACK); + init_pair(Color::Engaged, color(28, COLOR_GREEN), color(28, COLOR_GREEN)); + init_pair(Color::Green, color(34, COLOR_GREEN), COLOR_BLACK); initWindows(); @@ -125,6 +126,7 @@ void ConsoleUI::initWindows() { // set the title bar wbkgd(w[Win::Title], A_REVERSE); + werase(w[Win::Title]); // PDCurses applies a colorless background only to cells it clears itself mvwprintw(w[Win::Title], 0, 3, "openpilot replay %s", COMMA_VERSION); // show windows on the real screen @@ -139,16 +141,22 @@ void ConsoleUI::initWindows() { } void ConsoleUI::updateSize() { - if (is_term_resized(max_height, max_width)) { - for (auto win : w) { - if (win) delwin(win); - } - endwin(); - clear(); - refresh(); - initWindows(); - rWarning("resize term %dx%d", max_height, max_width); +#ifdef _WIN32 + // PDCurses keeps reporting the resize until resize_term() acknowledges it + if (!is_termresized()) return; + resize_term(0, 0); + if (LINES == max_height && COLS == max_width) return; +#else + if (!is_term_resized(max_height, max_width)) return; +#endif + for (auto win : w) { + if (win) delwin(win); } + endwin(); + clear(); + refresh(); + initWindows(); + rWarning("resize term %dx%d", max_height, max_width); } void ConsoleUI::updateStatus() { @@ -170,9 +178,9 @@ void ConsoleUI::updateStatus() { auto [status_str, status_color] = status_text[status]; write_item(0, 0, "STATUS: ", status_str, " ", false, status_color); auto cur_ts = replay->routeDateTime() + (int)replay->currentSeconds(); - char *time_string = ctime(&cur_ts); + const char *time_string = ctime(&cur_ts); // NULL for out-of-range times on Windows std::string current_segment = " - " + std::to_string((int)(replay->currentSeconds() / 60)); - write_item(0, 25, "TIME: ", time_string, current_segment, true); + write_item(0, 25, "TIME: ", time_string ? time_string : "?", current_segment, true); auto p = sm["vehicleParameters"].getVehicleParameters(); write_item(1, 0, "STIFFNESS: ", util::string_format("%.2f %%", p.getStiffnessFactor() * 100), " "); @@ -260,17 +268,19 @@ void ConsoleUI::updateTimeline() { for (const auto &entry : *replay->getTimeline()) { int start_pos = ((entry.start_time - replay->minSeconds()) / total_sec) * width; int end_pos = ((entry.end_time - replay->minSeconds()) / total_sec) * width; + // chgat keeps the cell's character and takes the color as a pair number, not as attribute bits; + // the markers use the same '_' as the legend if (entry.type == TimelineType::Engaged) { - mvwchgat(win, 1, start_pos, end_pos - start_pos + 1, A_COLOR, Color::Engaged, NULL); - mvwchgat(win, 2, start_pos, end_pos - start_pos + 1, A_COLOR, Color::Engaged, NULL); + mvwchgat(win, 1, start_pos, end_pos - start_pos + 1, A_NORMAL, Color::Engaged, NULL); + mvwchgat(win, 2, start_pos, end_pos - start_pos + 1, A_NORMAL, Color::Engaged, NULL); } else if (entry.type == TimelineType::UserBookmark) { - mvwchgat(win, 3, start_pos, end_pos - start_pos + 1, ACS_S3, Color::Cyan, NULL); + mvwhline(win, 3, start_pos, '_' | COLOR_PAIR(Color::Cyan), end_pos - start_pos + 1); } else { auto color_id = Color::Green; if (entry.type != TimelineType::AlertInfo) { color_id = entry.type == TimelineType::AlertWarning ? Color::Yellow : Color::Red; } - mvwchgat(win, 3, start_pos, end_pos - start_pos + 1, ACS_S3, color_id, NULL); + mvwhline(win, 3, start_pos, '_' | COLOR_PAIR(color_id), end_pos - start_pos + 1); } } diff --git a/openpilot/tools/replay/framereader.cc b/openpilot/tools/replay/framereader.cc index 19e5fa0d0f0800..5a624be38546f2 100644 --- a/openpilot/tools/replay/framereader.cc +++ b/openpilot/tools/replay/framereader.cc @@ -14,6 +14,9 @@ #ifdef __APPLE__ #define HW_DEVICE_TYPE AV_HWDEVICE_TYPE_VIDEOTOOLBOX #define HW_PIX_FMT AV_PIX_FMT_VIDEOTOOLBOX +#elif defined(_WIN32) +#define HW_DEVICE_TYPE AV_HWDEVICE_TYPE_D3D11VA +#define HW_PIX_FMT AV_PIX_FMT_D3D11 #else #define HW_DEVICE_TYPE AV_HWDEVICE_TYPE_CUDA #define HW_PIX_FMT AV_PIX_FMT_CUDA @@ -40,7 +43,7 @@ struct DecoderManager { } std::unique_ptr decoder; - #ifndef __APPLE__ + #if !defined(__APPLE__) && !defined(_WIN32) if (!Hardware::PC() && hw_decoder) { decoder = std::make_unique(); } else @@ -268,7 +271,7 @@ bool FFmpegVideoDecoder::copyBuffer(AVFrame *f, VisionBuf *buf) { return true; } -#ifndef __APPLE__ +#if !defined(__APPLE__) && !defined(_WIN32) bool V4LVideoDecoder::open(AVCodecParameters *codecpar, bool hw_decoder) { if (codecpar->codec_id != AV_CODEC_ID_HEVC) { rError("Hardware decoder only supports HEVC codec"); diff --git a/openpilot/tools/replay/framereader.h b/openpilot/tools/replay/framereader.h index 65feb5b3b72bc3..ca737aedff5a1a 100644 --- a/openpilot/tools/replay/framereader.h +++ b/openpilot/tools/replay/framereader.h @@ -6,7 +6,7 @@ #include "msgq/visionipc/visionbuf.h" #include "tools/replay/util.h" -#ifndef __APPLE__ +#if !defined(__APPLE__) && !defined(_WIN32) #include "system/loggerd/encoder/v4l_decoder.h" #endif @@ -66,7 +66,7 @@ class FFmpegVideoDecoder : public VideoDecoder { AVBufferRef *hw_device_ctx = nullptr; }; -#ifndef __APPLE__ +#if !defined(__APPLE__) && !defined(_WIN32) class V4LVideoDecoder : public VideoDecoder { public: V4LVideoDecoder() {}; diff --git a/openpilot/tools/replay/main.cc b/openpilot/tools/replay/main.cc index 50233189a1238d..29401f145b8fa8 100644 --- a/openpilot/tools/replay/main.cc +++ b/openpilot/tools/replay/main.cc @@ -9,6 +9,7 @@ #include "common/prefix.h" #include "common/timing.h" +#include "common/util.h" #include "tools/replay/consoleui.h" #include "tools/replay/replay.h" #include "tools/replay/util.h" @@ -139,7 +140,9 @@ int main(int argc, char *argv[]) { // The vendored ncurses static library has a wrong compiled-in terminfo path. // Point it at the system terminfo database if not already set. +#ifndef _WIN32 setenv("TERMINFO_DIRS", "/usr/share/terminfo:/lib/terminfo:/usr/lib/terminfo", 0); +#endif ReplayConfig config; diff --git a/openpilot/tools/replay/py_downloader.cc b/openpilot/tools/replay/py_downloader.cc index a7ab5baa917a38..5e926e7410c30e 100644 --- a/openpilot/tools/replay/py_downloader.cc +++ b/openpilot/tools/replay/py_downloader.cc @@ -5,14 +5,18 @@ #include #include #include +#include +#include +#ifdef _WIN32 +#include "common/win32.h" +#else #include #ifdef __APPLE__ #include #endif #include -#include #include -#include +#endif #include "tools/replay/util.h" @@ -30,6 +34,129 @@ void reportProgress(const char *line) { // Run a Python command and capture stdout. Stderr is scanned for PROGRESS lines and otherwise passed // through to the parent's stderr. Returns stdout content. If abort is signaled, kills the child process. +#ifdef _WIN32 +// CreateProcess with pipes instead of fork/exec; stderr still carries the progress lines +std::string runPython(const std::vector &args, std::atomic *abort = nullptr) { + std::string cmdline = "python3 -m openpilot.tools.lib.file_downloader"; + for (const auto &a : args) { + cmdline += " \"" + a + "\""; + } + + // Clear OPENPILOT_PREFIX for the child only so the Python process uses default + // paths (e.g. ~/.comma/auth.json). The prefix is only for IPC in the parent. + std::string env_block; + if (LPCH env = GetEnvironmentStringsA()) { + for (const char *p = env; *p; p += strlen(p) + 1) { + if (strncmp(p, "OPENPILOT_PREFIX=", 17) != 0) { + env_block.append(p, strlen(p) + 1); + } + } + FreeEnvironmentStringsA(env); + } + env_block.push_back('\0'); + + SECURITY_ATTRIBUTES sa = {}; + sa.nLength = sizeof(sa); + sa.bInheritHandle = TRUE; + HANDLE out_r = NULL, out_w = NULL, err_r = NULL, err_w = NULL; + if (!CreatePipe(&out_r, &out_w, &sa, 0) || !CreatePipe(&err_r, &err_w, &sa, 0)) { + rWarning("py_downloader: CreatePipe() failed"); + return {}; + } + SetHandleInformation(out_r, HANDLE_FLAG_INHERIT, 0); + SetHandleInformation(err_r, HANDLE_FLAG_INHERIT, 0); + + STARTUPINFOA si = {}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdOutput = out_w; + si.hStdError = err_w; + si.hStdInput = INVALID_HANDLE_VALUE; + PROCESS_INFORMATION pi = {}; + BOOL ok = CreateProcessA(NULL, cmdline.data(), NULL, NULL, TRUE, CREATE_NO_WINDOW, env_block.data(), NULL, &si, &pi); + CloseHandle(out_w); + CloseHandle(err_w); + if (!ok) { + CloseHandle(out_r); + CloseHandle(err_r); + rWarning("py_downloader: CreateProcess() failed: %lu", GetLastError()); + return {}; + } + CloseHandle(pi.hThread); + + std::thread stderr_thread([err_r]() { + std::string line; + char buf[4096]; + DWORD n = 0; + auto flush_line = [&line]() { + if (strncmp(line.c_str(), "PROGRESS:", 9) == 0) { + reportProgress(line.c_str()); + } else { + fprintf(stderr, "%s\n", line.c_str()); + } + line.clear(); + }; + while (ReadFile(err_r, buf, sizeof(buf), &n, NULL) && n > 0) { + for (DWORD i = 0; i < n; i++) { + if (buf[i] == '\n') { + flush_line(); + } else { + line.push_back(buf[i]); + } + } + } + if (!line.empty()) flush_line(); + CloseHandle(err_r); + }); + + std::string stdout_data; + char buf[4096]; + DWORD n = 0; + // Poll the pipe so abort can interrupt while waiting for Python output + while (true) { + if (abort && *abort) { + TerminateProcess(pi.hProcess, 1); + break; + } + DWORD avail = 0; + if (!PeekNamedPipe(out_r, NULL, 0, NULL, &avail, NULL)) break; // writer gone and pipe drained + if (avail == 0) { + if (WaitForSingleObject(pi.hProcess, 100) == WAIT_OBJECT_0) { + if (!PeekNamedPipe(out_r, NULL, 0, NULL, &avail, NULL) || avail == 0) break; + } + continue; + } + if (!ReadFile(out_r, buf, sizeof(buf), &n, NULL) || n == 0) break; + stdout_data.append(buf, n); + } + while (ReadFile(out_r, buf, sizeof(buf), &n, NULL) && n > 0) { + stdout_data.append(buf, n); + } + CloseHandle(out_r); + stderr_thread.join(); + + WaitForSingleObject(pi.hProcess, INFINITE); + DWORD code = 1; + GetExitCodeProcess(pi.hProcess, &code); + CloseHandle(pi.hProcess); + + const bool aborted = abort && *abort; + if (aborted || code != 0) { + if (!aborted) rWarning("py_downloader: process exited with code %lu", code); + std::lock_guard lk(handler_mutex); + if (progress_handler) { + progress_handler(0, 0, false); + } + return {}; + } + + // Trim trailing newline + while (!stdout_data.empty() && (stdout_data.back() == '\n' || stdout_data.back() == '\r')) { + stdout_data.pop_back(); + } + return stdout_data; +} +#else std::string runPython(const std::vector &args, std::atomic *abort = nullptr) { // Build argv for the downloader module std::vector argv; @@ -200,6 +327,7 @@ std::string runPython(const std::vector &args, std::atomic *a return stdout_data; } +#endif } // namespace diff --git a/openpilot/tools/replay/replay.cc b/openpilot/tools/replay/replay.cc index 26e10e7c7bbbdc..e0adbd1bc272e1 100644 --- a/openpilot/tools/replay/replay.cc +++ b/openpilot/tools/replay/replay.cc @@ -8,7 +8,9 @@ #include "common/params.h" #include "tools/replay/util.h" +#ifndef _WIN32 static void interrupt_sleep_handler(int signal) {} +#endif // Helper function to notify events with safety checks template @@ -19,7 +21,9 @@ void notifyEvent(Callback &callback, Args &&...args) { Replay::Replay(const std::string &route, std::vector allow, std::vector block, SubMaster *sm, uint32_t flags, const std::string &data_dir, bool auto_source) : sm_(sm), flags_(flags), seg_mgr_(std::make_unique(route, flags, data_dir, auto_source)) { +#ifndef _WIN32 std::signal(SIGUSR1, interrupt_sleep_handler); +#endif if (flags_ & REPLAY_FLAG_BENCHMARK) { benchmark_stats_.process_start_ts = nanos_since_boot(); @@ -104,9 +108,11 @@ bool Replay::load() { } void Replay::interruptStream(const std::function &update_fn) { +#ifndef _WIN32 if (stream_thread_.joinable() && stream_thread_id) { pthread_kill(stream_thread_id, SIGUSR1); // Interrupt sleep in stream thread } +#endif { interrupt_requested_ = true; std::unique_lock lock(stream_lock_); @@ -273,7 +279,9 @@ void Replay::publishFrame(const Event *e) { } void Replay::streamThread() { +#ifndef _WIN32 stream_thread_id = pthread_self(); +#endif std::unique_lock lk(stream_lock_); int last_processed_segment = -1; diff --git a/openpilot/tools/replay/replay.h b/openpilot/tools/replay/replay.h index 59e1d67d48a043..f769baced8083c 100644 --- a/openpilot/tools/replay/replay.h +++ b/openpilot/tools/replay/replay.h @@ -51,7 +51,8 @@ class Replay { void setLoop(bool loop) { loop ? flags_ &= ~REPLAY_FLAG_NO_LOOP : flags_ |= REPLAY_FLAG_NO_LOOP; } bool loop() const { return !(flags_ & REPLAY_FLAG_NO_LOOP); } const Route &route() const { return seg_mgr_->route_; } - inline double currentSeconds() const { return double(cur_mono_time_ - route_start_ts_) / 1e9; } + // signed: cur_mono_time_ starts 1 ns before route_start_ts_ so the first event is not skipped + inline double currentSeconds() const { return double(int64_t(cur_mono_time_ - route_start_ts_)) / 1e9; } inline std::time_t routeDateTime() const { return route_date_time_; } inline uint64_t routeStartNanos() const { return route_start_ts_; } inline double toSeconds(uint64_t mono_time) const { return (mono_time - route_start_ts_) / 1e9; } @@ -91,7 +92,9 @@ class Replay { std::unique_ptr seg_mgr_; Timeline timeline_; - pthread_t stream_thread_id = 0; +#ifndef _WIN32 + pthread_t stream_thread_id = 0; // Windows: precise_nano_sleep polls interrupt_requested_ instead of taking a signal +#endif std::thread stream_thread_; std::mutex stream_lock_; bool user_paused_ = false; diff --git a/openpilot/tools/replay/route.cc b/openpilot/tools/replay/route.cc index f38c2b19719381..58698a09f88ef4 100644 --- a/openpilot/tools/replay/route.cc +++ b/openpilot/tools/replay/route.cc @@ -1,5 +1,7 @@ #include "tools/replay/route.h" +#include "common/util.h" + #include #include #include diff --git a/openpilot/tools/replay/util.cc b/openpilot/tools/replay/util.cc index 44e144443abbd7..d8254de4614670 100644 --- a/openpilot/tools/replay/util.cc +++ b/openpilot/tools/replay/util.cc @@ -2,6 +2,10 @@ #include #include +#ifdef _WIN32 +#include +#include "common/win32.h" +#endif #include #include #include @@ -54,6 +58,28 @@ std::string getUrlWithoutQuery(const std::string &url) { return (idx == std::string::npos ? url : url.substr(0, idx)); } +#ifdef _WIN32 +// winpthreads' clock_nanosleep rejects CLOCK_MONOTONIC and Sleep() has a 15.6 ms granularity, so wait on a +// high-resolution timer. Nothing can cut the wait short, so sleep in slices and poll the interrupt flag between them. +void precise_nano_sleep(int64_t nanoseconds, std::atomic &interrupt_requested) { + struct Timer { + HANDLE h = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); + Timer() { if (!h) h = CreateWaitableTimerW(nullptr, TRUE, nullptr); } + ~Timer() { CloseHandle(h); } + }; + static thread_local Timer timer; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::nanoseconds(nanoseconds); + while (!interrupt_requested) { + const auto remaining = deadline - std::chrono::steady_clock::now(); + if (remaining <= std::chrono::nanoseconds::zero()) break; + const auto slice = std::min(remaining, std::chrono::milliseconds(5)); + LARGE_INTEGER due; + due.QuadPart = -std::max(slice.count() / 100, 1); // relative, 100 ns units + SetWaitableTimer(timer.h, &due, 0, nullptr, nullptr, FALSE); + WaitForSingleObject(timer.h, INFINITE); + } +} +#else void precise_nano_sleep(int64_t nanoseconds, std::atomic &interrupt_requested) { struct timespec req, rem; req.tv_sec = nanoseconds / 1000000000; @@ -72,6 +98,7 @@ void precise_nano_sleep(int64_t nanoseconds, std::atomic &interrupt_reques req = rem; } } +#endif std::vector split(std::string_view source, char delimiter) { std::vector fields; @@ -100,7 +127,11 @@ void *MonotonicBuffer::allocate(size_t bytes, size_t alignment) { void *p = std::align(alignment, bytes, current_buf, available); if (p == nullptr) { available = next_buffer_size = std::max(next_buffer_size, bytes); +#ifdef _WIN32 + current_buf = buffers.emplace_back(_aligned_malloc(next_buffer_size, alignment)); // no aligned_alloc in the Windows CRT +#else current_buf = buffers.emplace_back(std::aligned_alloc(alignment, next_buffer_size)); +#endif next_buffer_size *= growth_factor; p = current_buf; } @@ -112,6 +143,10 @@ void *MonotonicBuffer::allocate(size_t bytes, size_t alignment) { MonotonicBuffer::~MonotonicBuffer() { for (auto buf : buffers) { +#ifdef _WIN32 + _aligned_free(buf); +#else free(buf); +#endif } } diff --git a/panda b/panda index 75aa44bec91408..1d4538e3ed27f9 160000 --- a/panda +++ b/panda @@ -1 +1 @@ -Subproject commit 75aa44bec9140849868239b1f1e3f22624adb8fe +Subproject commit 1d4538e3ed27f98313a5ebf05a1f45928f933ab6 diff --git a/pyproject.toml b/pyproject.toml index 7be8f97c0f890e..e2f4fd402cdfbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "comma-deps-zstd", "comma-deps-zeromq", "comma-deps-json11", + "comma-deps-eigen", # rednose; listed here so uv applies the Windows source to it "comma-deps-git-lfs", "comma-deps-gcc-arm-none-eabi", @@ -78,6 +79,7 @@ submodules = [ "pandacan", "rednose", "teleoprtc", + "libdatachannel-py; sys_platform == 'win32'", # teleoprtc's dependency; uv sources only apply to direct dependencies "tinygrad", ] @@ -154,6 +156,14 @@ override-dependencies = [ "opendbc", # panda pins opendbc from git for standalone use; always use our submodule ] +# --- TODO REMOVE AFTER DEPENDENCY PR MERGES (commaai/dependencies#107): fork-only index of the comma-deps Windows wheels until PyPI has them; not for upstream --- +# (publish_windows.sh there). Point at a local `dist/` directory instead to use wheels built with its build.sh. +[[tool.uv.index]] +name = "comma-deps-windows" +url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" +format = "flat" +explicit = true + [tool.uv.sources] msgq = { path = "msgq_repo", editable = true } opendbc = { path = "opendbc_repo", editable = true } @@ -161,3 +171,18 @@ pandacan = { path = "panda", editable = true } rednose = { path = "rednose_repo", editable = true } teleoprtc = { path = "teleoprtc_repo", editable = true } tinygrad = { path = "tinygrad_repo", editable = true } +libdatachannel-py = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } # --- FORK ONLY, NOT FOR UPSTREAM: teleoprtc on Windows from the fork index until libdatachannel-py publishes win_amd64 wheels --- +comma-deps-acados = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-bootstrap-icons = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-capnproto = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-eigen = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-ffmpeg = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-gcc-arm-none-eabi = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-git-lfs = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-imgui = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-json11 = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-libusb = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-ncurses = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-raylib = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-zeromq = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } +comma-deps-zstd = { index = "comma-deps-windows", marker = "sys_platform == 'win32'" } diff --git a/rednose_repo b/rednose_repo index 28d4a7f69e80e1..856bad5a5cde4e 160000 --- a/rednose_repo +++ b/rednose_repo @@ -1 +1 @@ -Subproject commit 28d4a7f69e80e1c3e0d24ca0733d7daeaeade3d0 +Subproject commit 856bad5a5cde4e26179ce3f2992ff48ff229ff02 diff --git a/tools/README.md b/tools/README.md index ae36282828fcea..9ef809e37b8a00 100644 --- a/tools/README.md +++ b/tools/README.md @@ -4,7 +4,7 @@ openpilot is developed and tested on **Ubuntu 24.04**, which is the primary development target aside from the [supported embedded hardware](https://github.com/commaai/openpilot#running-on-a-dedicated-device-in-a-car). -Most of openpilot should work natively on macOS. On Windows you can use WSL for a nearly native Ubuntu experience. Running natively on any other system is not currently recommended and will likely require modifications. +Most of openpilot should work natively on macOS and, for development only, on Windows. On Windows you can also use WSL for a nearly native Ubuntu experience. Running natively on any other system is not currently recommended and will likely require modifications. ## Native setup on Ubuntu 24.04 and macOS @@ -32,6 +32,34 @@ source .venv/bin/activate scons -u ``` +## Native setup on Windows + +The development tools (UI, cabana, replay, jotpluggler, the models and the unit tests) build natively on Windows; there is no on-road support and the comma device processes that need Linux do not run, exactly as on macOS. The build runs in an [MSYS2](https://www.msys2.org/) CLANG64 shell (clang, lld and libc++) with a native Python managed by uv, so it behaves like the macOS setup rather than WSL. + +**1. Install Git for Windows and MSYS2, then clone openpilot** from MSYS2's CLANG64 shell started with the Windows PATH (`clang64.exe -full-path`, or `MSYS2_PATH_TYPE=inherit`) so that Git for Windows is the git in it; do not install MSYS2's own `git` package, git-lfs cannot drive it. Scripts and patches must stay LF, in the submodules too. +``` bash +git config --global core.autocrlf false +git clone https://github.com/commaai/openpilot.git +``` + +**2. Run the setup script** from that shell. It installs the toolchain with pacman, uv, the Python dependencies (comma's dependencies come as prebuilt Windows wheels) and the LFS files. +``` bash +cd openpilot +tools/op.sh setup +``` + +**3. Activate a Python shell** +``` bash +source .venv/Scripts/activate +``` + +**4. Build openpilot** +``` bash +scons -u +``` + +The tools run from the same shell, e.g. `openpilot/tools/cabana/cabana --demo` or `python openpilot/selfdrive/ui/ui.py`, and `tools/op.sh test` runs the unit tests. + ## WSL on Windows [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/about) should provide a similar experience to native Ubuntu. [WSL 2](https://docs.microsoft.com/en-us/windows/wsl/compare-versions) specifically has been reported by several users to be a seamless experience. diff --git a/tools/op.sh b/tools/op.sh index f17714e620d259..3066fa43b357a8 100755 --- a/tools/op.sh +++ b/tools/op.sh @@ -20,6 +20,13 @@ if [ "$(uname)" == "Darwin" ] && [ $SHELL == "/bin/bash" ]; then RC_FILE="$HOME/.bash_profile" fi +# Windows builds run in an MSYS2 CLANG64 shell with a native Python, whose venv keeps its scripts in Scripts/ +VENV_BIN="bin" +if [[ "$(uname -s)" == MINGW* || "$(uname -s)" == MSYS* ]]; then + VENV_BIN="Scripts" + export PYTHONUTF8=1 # redirected output would use the ANSI code page otherwise +fi + function retry() { local attempts=$1 shift @@ -126,6 +133,12 @@ function op_check_os() { echo -e " ↳ [${GREEN}✔${NC}] Linux detected." elif [[ "$OSTYPE" == "darwin"* ]]; then echo -e " ↳ [${GREEN}✔${NC}] macOS detected." + elif [[ "$(uname -s)" == MINGW* || "$(uname -s)" == MSYS* ]]; then + if [[ "${MSYSTEM:-}" != "CLANG64" ]]; then + echo -e " ↳ [${RED}✗${NC}] Windows needs an MSYS2 CLANG64 shell, this is ${MSYSTEM:-not MSYS2}!" + return 1 + fi + echo -e " ↳ [${GREEN}✔${NC}] Windows (MSYS2 CLANG64) detected." else echo -e " ↳ [${RED}✗${NC}] OS type $OSTYPE not supported!" return 1 @@ -134,7 +147,7 @@ function op_check_os() { function op_check_venv() { echo "Checking for venv..." - if [[ -f $OPENPILOT_ROOT/.venv/bin/activate ]]; then + if [[ -f $OPENPILOT_ROOT/.venv/$VENV_BIN/activate ]]; then echo -e " ↳ [${GREEN}✔${NC}] venv detected." else echo -e " ↳ [${RED}✗${NC}] Can't activate venv in $OPENPILOT_ROOT. Assuming global env!" @@ -206,11 +219,11 @@ EOF echo "Pulling git lfs files..." st="$(date +%s)" - git config --local filter.lfs.clean ".venv/bin/git-lfs clean -- %f" - git config --local filter.lfs.smudge ".venv/bin/git-lfs smudge -- %f" - git config --local filter.lfs.process ".venv/bin/git-lfs filter-process" + git config --local filter.lfs.clean ".venv/$VENV_BIN/git-lfs clean -- %f" + git config --local filter.lfs.smudge ".venv/$VENV_BIN/git-lfs smudge -- %f" + git config --local filter.lfs.process ".venv/$VENV_BIN/git-lfs filter-process" git config --local filter.lfs.required true - printf '#!/bin/sh\nexec .venv/bin/git-lfs pre-push "$@"\n' > "$(git rev-parse --git-path hooks)/pre-push" + printf '#!/bin/sh\nexec .venv/%s/git-lfs pre-push "$@"\n' "$VENV_BIN" > "$(git rev-parse --git-path hooks)/pre-push" chmod +x "$(git rev-parse --git-path hooks)/pre-push" if ! retry 3 git lfs pull; then echo -e " ↳ [${RED}✗${NC}] Pulling git lfs files failed!" @@ -230,19 +243,21 @@ function op_auth() { function op_activate_venv() { # bash 3.2 can't handle this without the 'set +e' set +e - source $OPENPILOT_ROOT/.venv/bin/activate &> /dev/null || true + source $OPENPILOT_ROOT/.venv/$VENV_BIN/activate &> /dev/null || true set -e # persist venv on PATH across GitHub Actions steps if [ -n "$GITHUB_PATH" ]; then - echo "$OPENPILOT_ROOT/.venv/bin" >> "$GITHUB_PATH" + VENV_PATH="$OPENPILOT_ROOT/.venv/$VENV_BIN" + command -v cygpath > /dev/null && VENV_PATH="$(cygpath -w "$VENV_PATH")" # the runner's PATH is a Windows one + echo "$VENV_PATH" >> "$GITHUB_PATH" fi } function op_venv() { op_before_cmd - if [[ ! -f $OPENPILOT_ROOT/.venv/bin/activate ]]; then + if [[ ! -f $OPENPILOT_ROOT/.venv/$VENV_BIN/activate ]]; then echo -e "No venv found in $OPENPILOT_ROOT" return 1 fi @@ -250,10 +265,10 @@ function op_venv() { case $SHELL_NAME in "zsh") ZSHRC_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'tmp_zsh') - echo "source $RC_FILE; source $OPENPILOT_ROOT/.venv/bin/activate" >> $ZSHRC_DIR/.zshrc + echo "source $RC_FILE; source $OPENPILOT_ROOT/.venv/$VENV_BIN/activate" >> $ZSHRC_DIR/.zshrc ZDOTDIR=$ZSHRC_DIR zsh ;; *) - bash --rcfile <(echo "source $RC_FILE; source $OPENPILOT_ROOT/.venv/bin/activate") ;; + bash --rcfile <(echo "source $RC_FILE; source $OPENPILOT_ROOT/.venv/$VENV_BIN/activate") ;; esac } diff --git a/tools/setup_dependencies.sh b/tools/setup_dependencies.sh index 5ad833a5cae0ec..eaab6383524eaf 100755 --- a/tools/setup_dependencies.sh +++ b/tools/setup_dependencies.sh @@ -4,6 +4,9 @@ set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" ROOT="$(git -C "$DIR" rev-parse --show-toplevel)" +VENV_BIN="bin" +case "$(uname -s)" in MINGW*|MSYS*) VENV_BIN="Scripts" ;; esac # native Python venv layout on Windows + function retry() { local attempts=$1 shift @@ -93,6 +96,22 @@ function install_linux_deps() { fi } +function install_windows_deps() { + if [[ "${MSYSTEM:-}" != "CLANG64" ]]; then + echo "Windows builds need an MSYS2 CLANG64 shell, this is ${MSYSTEM:-not MSYS2}" + exit 1 + fi + # git-lfs (a native binary from the venv) cannot drive MSYS2's Cygwin-style git; Git for Windows is the git here + if [[ "$(command -v git)" == /usr/bin/git ]]; then + echo "MSYS2's git package shadows Git for Windows; remove it (pacman -R git) and start the shell with the Windows PATH (clang64.exe -full-path)" + exit 1 + fi + # clang, lld and libc++ (MSVC cannot build openpilot's GNU C); dlfcn provides dlopen + pacman -S --needed --noconfirm \ + "$MINGW_PACKAGE_PREFIX-toolchain" "$MINGW_PACKAGE_PREFIX-pkgconf" "$MINGW_PACKAGE_PREFIX-ccache" \ + "$MINGW_PACKAGE_PREFIX-dlfcn" file unzip # unzip: the uv installer +} + function install_python_deps() { # Increase the pip timeout to handle TimeoutError export PIP_DEFAULT_TIMEOUT=200 @@ -113,7 +132,7 @@ function install_python_deps() { echo "installing python packages..." uv sync --frozen --all-extras - source .venv/bin/activate + source .venv/$VENV_BIN/activate } # --- Main --- @@ -127,6 +146,9 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then elif [[ $SHELL == "/bin/bash" ]]; then RC_FILE="$HOME/.bash_profile" fi +elif [[ "$(uname -s)" == MINGW* || "$(uname -s)" == MSYS* ]]; then + install_windows_deps + echo "[ ] installed system dependencies t=$SECONDS" fi if [ -f "$ROOT/pyproject.toml" ]; then diff --git a/tools/test_runner.py b/tools/test_runner.py index 1979ce01e46843..963600f9287b43 100755 --- a/tools/test_runner.py +++ b/tools/test_runner.py @@ -140,6 +140,7 @@ def collect(targets, keyword): loader = unittest.TestLoader() tests = [] errors = [] + skipped = [] names = [] for target in targets: path_text, *nodes = target.split("::") @@ -162,6 +163,9 @@ def collect(targets, keyword): before = len(loader.errors) try: suite = loader.loadTestsFromName(name) + except unittest.SkipTest: # raised at import for a whole module, as unittest discovery allows + skipped.append(make_record(name, "skipped")) + continue except Exception: errors.append(f"Failed to collect {name}\n{traceback.format_exc()}") continue @@ -174,7 +178,7 @@ def collect(targets, keyword): continue if not keyword or keyword.lower() in test.id().lower(): tests.append(test) - return list({test.id(): test for test in tests}.values()), errors + return list({test.id(): test for test in tests}.values()), errors, skipped def make_batches(tests, workers): @@ -272,13 +276,13 @@ def main(): os.chdir(ROOT) warnings.simplefilter(args.warnings) started = time.monotonic() - tests, errors = collect(args.targets, args.k) + tests, errors, skipped = collect(args.targets, args.k) batches = make_batches(tests, args.jobs) workers = min(args.jobs, len(batches)) summary = f"collected {len(tests)} test{'s' if len(tests) != 1 else ''} in {time.monotonic() - started:.2f}s " summary += f"• {workers} worker{'s' if workers != 1 else ''}" print(summary) - records = [] + records = list(skipped) column = 0 try: if workers < 2: @@ -290,7 +294,7 @@ def main(): for item in batch: mark, code = STATUS_MARKS[item["status"]] if args.verbose: - print(f"{paint(mark, code)} {item['id']} {item['time']:.2f}s") + print(f"{paint(mark, code)} {item['id']} {item['time']:.2f}s", flush=True) else: print(paint(mark, code), end="", flush=True) column += 1 diff --git a/uv.lock b/uv.lock index c27429f5f209c5..5b515de10a6a61 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,10 @@ version = 1 revision = 3 requires-python = ">=3.12.3, <3.13" +resolution-markers = [ + "sys_platform == 'win32'", + "sys_platform != 'win32'", +] [manifest] overrides = [{ name = "opendbc", editable = "opendbc_repo" }] @@ -100,8 +104,11 @@ wheels = [ name = "comma-deps-acados" version = "0.2.2.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/3d/13/1190aed06e91a9f9024b16fb44a4184842e56ac39dbaec8e6aea83cb1d7e/comma_deps_acados-0.2.2.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:64c002e0d6170c7bdec300159bbbe07cf3e05fa54ae5890fe736190fc3543fe7", size = 10635996, upload-time = "2026-07-23T17:01:04.136Z" }, @@ -109,110 +116,267 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/9d/24377b731093e015a44fff043dd7ea5b77b0de62acf48b5a0e7d5a662a15/comma_deps_acados-0.2.2.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:e55ac429d848415930a0b82ab100a310e1848b48cdbaa8dda574b561b43c50d0", size = 13124767, upload-time = "2026-07-23T17:01:13.091Z" }, ] +[[package]] +name = "comma-deps-acados" +version = "0.2.2.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +dependencies = [ + { name = "numpy", marker = "sys_platform == 'win32'" }, +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_acados-0.2.2.post115-py3-none-win_amd64.whl", hash = "sha256:e45451d210287fc3165094e9c9be0d10997d89a50f32b5c0a8f5f2f98df066fb" }, +] + [[package]] name = "comma-deps-bootstrap-icons" version = "1.10.5.0.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/aa/69/da1a72b8b7783b0caf9a54b27c7124bad11768b8bce2c656ef3b700ab831/comma_deps_bootstrap_icons-1.10.5.0.post98-py3-none-any.whl", hash = "sha256:cabaeecea398eb867b96a6c653c6078691a437c0eff2364530194a218d94cb99", size = 385998, upload-time = "2026-07-23T17:01:17.476Z" }, ] +[[package]] +name = "comma-deps-bootstrap-icons" +version = "1.10.5.0.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_bootstrap_icons-1.10.5.0.post115-py3-none-any.whl", hash = "sha256:ee2e13b8445e49f5a282e027e9f22ecfa4c2033db5fb2a0ffc61e9282280f2dc" }, +] + [[package]] name = "comma-deps-capnproto" version = "1.0.1.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/ca/83/d3e6346a31491be1d378e4585f37a7979eb772018616abfa74fb27750f1e/comma_deps_capnproto-1.0.1.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9f4d08682df92411b360bec855cb6475990313cd0ecd8ed5c6ee02befb9db913", size = 2407343, upload-time = "2026-07-23T17:01:21.247Z" }, { url = "https://files.pythonhosted.org/packages/b0/8b/6f2a29d50ed4c8741dbf0a34ab109899268d09753518cd693e881bbf1a9d/comma_deps_capnproto-1.0.1.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:6cdf838a8d415ac71e1f52306624ab3ab6f27777f6ce89c0059c4129ea7b7f62", size = 2506355, upload-time = "2026-07-23T17:01:25.254Z" }, { url = "https://files.pythonhosted.org/packages/08/24/e91f2203d62e4db9de7dae06dd0cdefb1e000d8b3ba0bde48367be7e5b63/comma_deps_capnproto-1.0.1.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:3d95993c9aff0c89e39ca965e995021dd3dccdfc3d5d85152916cf4bf651b7ec", size = 2590764, upload-time = "2026-07-23T17:01:29.062Z" }, ] +[[package]] +name = "comma-deps-capnproto" +version = "1.0.1.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_capnproto-1.0.1.post115-py3-none-win_amd64.whl", hash = "sha256:f4d08090b45ab2604d37d8b1ea639e4d2c5c0eafb4025ae4e3a82ace9a729fca" }, +] + [[package]] name = "comma-deps-eigen" version = "3.4.0.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/3e/2f/89011c71976da6e1c3d7be315afa3d86ff25deeada1ad2319ac6be0e18ea/comma_deps_eigen-3.4.0.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:bd182634cf4fa537e7815238d3135e6e89be5826421a495be92258c6c388b527", size = 2275893, upload-time = "2026-07-23T17:01:44.622Z" }, { url = "https://files.pythonhosted.org/packages/2c/61/fcd4ad536c51437ee73ac255f3b8a23fb5f21bb1f96e834d8036c3bbcf08/comma_deps_eigen-3.4.0.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:250c15f6217c37736a2f54298d01f6e32f6e977faa87cc358de67ea3d121725e", size = 2275896, upload-time = "2026-07-23T17:01:48.397Z" }, { url = "https://files.pythonhosted.org/packages/d3/a2/2b7633fe5a5a2914900933393c315e9bd86e8fb7bbbe328d3a220eaf2027/comma_deps_eigen-3.4.0.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:dee9eb6c6c7e58201d7a36611857b7b3f3ba70f888ee07493fef2ea41d0d2cae", size = 2275898, upload-time = "2026-07-23T17:01:52.179Z" }, ] +[[package]] +name = "comma-deps-eigen" +version = "3.4.0.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_eigen-3.4.0.post115-py3-none-win_amd64.whl", hash = "sha256:1f3690949c7ad5e98e5a2ddd16d66fdfa022118f28ad5e5cd819679045bcf8c2" }, +] + [[package]] name = "comma-deps-ffmpeg" version = "7.1.0.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/20/59/4899ac0fa54905f43e237fff122008d6c591918b41e36880eeb18cd6279c/comma_deps_ffmpeg-7.1.0.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7816c5adc9c6a7462209ccf1d023c42d7e38a4eee47aa2f43d45dfc8320063f8", size = 7326312, upload-time = "2026-07-23T17:01:55.975Z" }, { url = "https://files.pythonhosted.org/packages/29/cb/6e047c19c39977c5ae322ad698b91d8d9fce43314cb86563de91bb161982/comma_deps_ffmpeg-7.1.0.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:45ad401b4058e3f7efb8d6841e1f187e8c265e942e56e7fb01db1ba9096e6b78", size = 4437675, upload-time = "2026-07-23T17:01:59.971Z" }, { url = "https://files.pythonhosted.org/packages/76/3d/cda4b19fa5a7b26921a518143c94fd3632030a34b157cab6d6f10f2c86bc/comma_deps_ffmpeg-7.1.0.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:9e7034739d45a45254c4200a555d343b22ce117429bebc2de2c1b05f050bfc8c", size = 4681499, upload-time = "2026-07-23T17:02:03.963Z" }, ] +[[package]] +name = "comma-deps-ffmpeg" +version = "7.1.0.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_ffmpeg-7.1.0.post115-py3-none-win_amd64.whl", hash = "sha256:f008b3706f451c5d5ea8c82e924ca7dd2d1011912e8f5d95fb7010d010a36ce4" }, +] + [[package]] name = "comma-deps-gcc-arm-none-eabi" version = "13.2.1.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/0e/e5/a4cd9faa80bf419c6a7052c99dfe565c283a5c966e90ce35b1b4040b24b8/comma_deps_gcc_arm_none_eabi-13.2.1.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d0e6991b845636ab19e46199bc5cb9dd056611bc6b93c6bca4f2bb5002783533", size = 15238810, upload-time = "2026-07-23T17:02:08.588Z" }, { url = "https://files.pythonhosted.org/packages/5b/81/690ce48945aecf58e475cb728a8d2f6c034493afd87b5b0381a85dd324b4/comma_deps_gcc_arm_none_eabi-13.2.1.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:41fef00d033e0f6c12e1748829d95e53942826e0085f57d918351b2de69530d7", size = 17367240, upload-time = "2026-07-23T17:02:13.637Z" }, { url = "https://files.pythonhosted.org/packages/41/a9/6af914145bd5c9ce3468a95500558bdc0a438f69700daedd37535945294e/comma_deps_gcc_arm_none_eabi-13.2.1.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:ed3630aac06b3a1db78ba5a29a33b900a51dd1c0b37f86cbfe3c1591993178f8", size = 16941137, upload-time = "2026-07-23T17:02:18.976Z" }, ] +[[package]] +name = "comma-deps-gcc-arm-none-eabi" +version = "13.2.1.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_gcc_arm_none_eabi-13.2.1.post115-py3-none-win_amd64.whl", hash = "sha256:8058387197b73432db8645170550ac2d8e0624d0c2741f6857ff6f195d222910" }, +] + [[package]] name = "comma-deps-git-lfs" version = "3.6.1.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/79/27/ecfda511eb334822d9bc464ec2d9b74d3c553784811a885baba34a27eaf6/comma_deps_git_lfs-3.6.1.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:259b1f4859bb3ab20fdcc012be0a9868a3649e1087d5cec07eaade7adea44c78", size = 4685104, upload-time = "2026-07-23T17:02:23.67Z" }, { url = "https://files.pythonhosted.org/packages/00/a5/9631b4a676279b353f82d4e2da62eb567c70fff628133a62d7f70fbf5924/comma_deps_git_lfs-3.6.1.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:60d39254138b2c7c3f15cc512c14504c11881e79885492321e1ffc3ecb840f93", size = 4485276, upload-time = "2026-07-23T17:02:27.751Z" }, { url = "https://files.pythonhosted.org/packages/09/5a/7ef6bc209d8ec15c40b1f988345e2e59c535a215284366916422ba0d0c30/comma_deps_git_lfs-3.6.1.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:9586057ca9c6e77e9068128f3e96f0ca03db29a757414b1b251bab4ca31ee6b8", size = 4889582, upload-time = "2026-07-23T17:02:31.454Z" }, ] +[[package]] +name = "comma-deps-git-lfs" +version = "3.6.1.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_git_lfs-3.6.1.post115-py3-none-win_amd64.whl", hash = "sha256:9e0a58c236edea6aa2a3d90f77cf1a62b1a368ae2e20da9ef7ec2c4d7f452b82" }, +] + [[package]] name = "comma-deps-imgui" version = "1.92.7.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/f4/e4/b9f4b68973bfd529314c28fcd87cb2f52b5dc7d9fdeb3be2d3d15b7cea25/comma_deps_imgui-1.92.7.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6fddf76138b1e54fe33e9f5ea3cbcb650f92fef8fbcf7e5080800ea851d68b98", size = 1688011, upload-time = "2026-07-23T17:02:35.416Z" }, { url = "https://files.pythonhosted.org/packages/7f/46/92030abf6e42e9813f144d10bcf541b39a246c5ca2d63d049478deac650b/comma_deps_imgui-1.92.7.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9f7eed0f759e59afcf289967edb3d94c48a38fe97527f2909dbbc70a850dc2cc", size = 2522785, upload-time = "2026-07-23T17:02:39.092Z" }, { url = "https://files.pythonhosted.org/packages/7d/57/d41e76559a553565413976695fb63a768d3446d12eccc9e736a12b53e662/comma_deps_imgui-1.92.7.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:07eeb105cce73ec3b27789501c78dcd059d75e736a99f1953358ef5097e7036f", size = 2655476, upload-time = "2026-07-23T17:02:42.925Z" }, ] +[[package]] +name = "comma-deps-imgui" +version = "1.92.7.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_imgui-1.92.7.post115-py3-none-win_amd64.whl", hash = "sha256:a9d3711aa81df7f7ab7082a1e2a0f0a701d3e3061db03ce463f3493e00f391a8" }, +] + [[package]] name = "comma-deps-json11" version = "20170411.0.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/c1/f4/50411c9134a8347831a72f90318b7b7d91ce566e63575b8a4a821be50ca4/comma_deps_json11-20170411.0.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:20d666897062487e4cd93b8e4eb9c53ecabf706864be8d8cbb60a56f0113c452", size = 34034, upload-time = "2026-07-23T17:02:46.595Z" }, { url = "https://files.pythonhosted.org/packages/1f/54/0c87fae682ee52e6aec371336ac980921ad34cedabb69e576cc9f83c40a7/comma_deps_json11-20170411.0.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9b4a03909609b832dc99b06503fb8c77419399a99e0db982d4c3f51f1563aa73", size = 41848, upload-time = "2026-07-23T17:02:50.039Z" }, { url = "https://files.pythonhosted.org/packages/7b/71/dd100992e13f2c7a01f68eebcd1e3cf43f1d169e4b80b0577f330e5f5c12/comma_deps_json11-20170411.0.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:1da9030908f3a6631a0a254f493c382ba061c2e8ddb280a30d64430af29f4638", size = 42602, upload-time = "2026-07-23T17:02:53.233Z" }, ] +[[package]] +name = "comma-deps-json11" +version = "20170411.0.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_json11-20170411.0.post115-py3-none-win_amd64.whl", hash = "sha256:64397f5d1b2c24a682f140a76be5e8cd05a0f38a80d59fcdd525471f4a6bb654" }, +] + [[package]] name = "comma-deps-libusb" version = "1.0.29.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/47/fb/f7d342a8f785fc1c0fd5d6883e1a5a7d424a1899b888f78f9091f4b98049/comma_deps_libusb-1.0.29.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a10b82a946c33c23152cee3e330ce76d398ddce1f38fea62e33773bef8f56164", size = 102339, upload-time = "2026-07-23T17:02:56.567Z" }, { url = "https://files.pythonhosted.org/packages/b7/fe/1b21692cc03078219a3946aae56086a109168a4b4dcfba3a22ce1cd01064/comma_deps_libusb-1.0.29.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:39567eeef6170ece389526780f90c3b75cbcfdf427f648f6b1539c203f7387a8", size = 94431, upload-time = "2026-07-23T17:03:00.01Z" }, { url = "https://files.pythonhosted.org/packages/49/d2/d93aac76b94ae87f7a37ce88f2e7e1184e19e67d10c068c1aac209075450/comma_deps_libusb-1.0.29.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:5e5b86be94b4a355c6be933ed21586d01b1d7ba597298dafd33b871a1ae66416", size = 93462, upload-time = "2026-07-23T17:03:03.218Z" }, ] +[[package]] +name = "comma-deps-libusb" +version = "1.0.29.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_libusb-1.0.29.post115-py3-none-win_amd64.whl", hash = "sha256:9ae8c81cb8e5c2fd721f8a2979f84aadb5b49a20ca25626a40e40cb98c484bba" }, +] + [[package]] name = "comma-deps-ncurses" version = "6.5.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/d0/d4/03e62b2a0be92ad420653ff1cf4396de9840c4c59cdc6e000ea5614f7744/comma_deps_ncurses-6.5.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:22ade1596deb18538d4bf59708aa2747c28c2e0e7048f13f5bfce3e5588f7417", size = 264921, upload-time = "2026-07-23T17:03:06.576Z" }, { url = "https://files.pythonhosted.org/packages/1a/ab/295b428ef473dbe0d7088ff02daf5ba17b924f3b915ac69a8a9c45a6eb2b/comma_deps_ncurses-6.5.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:6724d3e8c1f2e59d2475f7588d86494bcc4001e993fd7dff4c91ecb046edc97f", size = 260844, upload-time = "2026-07-23T17:03:10.329Z" }, { url = "https://files.pythonhosted.org/packages/cd/db/75afb33eaa86425d9bee68153f6141be2645cf5699b2cab5a7cbf7a36099/comma_deps_ncurses-6.5.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:d85e70e98f4b0a969d63a4a61a1a5b85db3c648ea58422401b55f386813f7d12", size = 248352, upload-time = "2026-07-23T17:03:13.786Z" }, ] +[[package]] +name = "comma-deps-ncurses" +version = "6.5.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_ncurses-6.5.post115-py3-none-win_amd64.whl", hash = "sha256:be7012929a138b18abaeed4225c27faa0ae5d2f70c32c42c6ed0b3110d7cdbb4" }, +] + [[package]] name = "comma-deps-raylib" version = "6.0.0.1.post101" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] dependencies = [ - { name = "cffi" }, + { name = "cffi", marker = "sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ff/90/e289acd1725d71c792c33399422f1052c4b0c38aeb4222a866d30c4a2cad/comma_deps_raylib-6.0.0.1.post101-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fa69d5093a92d7d2bfd2714a1afccca63b94d4c55fae88e61b80e8841de6a6cd", size = 1885392, upload-time = "2026-08-27T18:24:37.25Z" }, @@ -220,26 +384,68 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/ca/ef33aff790b37dfc925f94fbb3a86da19f67929c15b3725ddb18afb91e96/comma_deps_raylib-6.0.0.1.post101-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:8f2a1ffe5f60cac06170b144d6ce84925c91657e7ada3ae35bc5df6ccbe0b461", size = 20722616, upload-time = "2026-08-27T18:24:45.803Z" }, ] +[[package]] +name = "comma-deps-raylib" +version = "6.0.0.1.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +dependencies = [ + { name = "cffi", marker = "sys_platform == 'win32'" }, +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_raylib-6.0.0.1.post115-py3-none-win_amd64.whl", hash = "sha256:f9d71d4a91ff16a4ddf8ce18425f1fb336477031e0c3aad0bb18c045c48c26ee" }, +] + [[package]] name = "comma-deps-zeromq" version = "4.3.5.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/12/b7/b0070e091dae4be2cecccfb2921167b568d0e7bb9ea600b5814603e0590f/comma_deps_zeromq-4.3.5.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:0ecde97133d657024bf99ac7302130905a131dabedbc8425b6666b2058ce6acb", size = 815150, upload-time = "2026-07-23T17:03:29.517Z" }, { url = "https://files.pythonhosted.org/packages/2f/9a/d6a381b079516eca1b8a86aa3e972e550a48ff2f069ccc722b118ab53d60/comma_deps_zeromq-4.3.5.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:110a628ea440ea75707ad29fd90341d300414b0092137b1d43e8f19d100cf2fa", size = 833389, upload-time = "2026-07-23T17:03:33.25Z" }, { url = "https://files.pythonhosted.org/packages/af/d7/504649efc8dbe8ce4c0cbec085178d1c3298f6950bd4bbca1683a90c49ed/comma_deps_zeromq-4.3.5.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:cd16a515f00f5fb679c2883970c2e7f446ad84e65d7fcf045d325952f9cc3607", size = 798894, upload-time = "2026-07-23T17:03:36.788Z" }, ] +[[package]] +name = "comma-deps-zeromq" +version = "4.3.5.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_zeromq-4.3.5.post115-py3-none-win_amd64.whl", hash = "sha256:7367aa7e9e701de22d28164ec5ec31154224584e37eab41a08a9a589bcec389c" }, +] + [[package]] name = "comma-deps-zstd" version = "1.5.6.post98" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/ef/66/fd1098b4514e759d85e19d604d446ecec1f67e2f452df9e280d01a2449f7/comma_deps_zstd-1.5.6.post98-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2b6acdd50e71ec67a1426423cda5116a7cb43900dd2e4fca2cbcbb7d588be171", size = 1065140, upload-time = "2026-07-23T17:03:40.465Z" }, { url = "https://files.pythonhosted.org/packages/0c/ba/1d61aae97577bbf13c9c02e7e69d4c9391947d8d0d09ca4ad78f2e3d0faa/comma_deps_zstd-1.5.6.post98-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:04825faf902754a0945ebac374d01d676b7e5f98a68e460452319de509b369f3", size = 1006145, upload-time = "2026-07-23T17:03:44.144Z" }, { url = "https://files.pythonhosted.org/packages/a9/22/94d164407b579090eb3aceeeb63fbd8c540f6972a11e5b72fe3ca3139333/comma_deps_zstd-1.5.6.post98-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:316200a52c9ac1aeb6b22030480ffebaa1d91756c18a3f3cf216368a5fb35bfd", size = 1030359, upload-time = "2026-07-23T17:03:47.999Z" }, ] +[[package]] +name = "comma-deps-zstd" +version = "1.5.6.post115" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/comma_deps_zstd-1.5.6.post115-py3-none-win_amd64.whl", hash = "sha256:4c164b0c4cf8bcb1eadbf9c54d8cfbfd088e38de209bd3390791ccf5fa8ba789" }, +] + [[package]] name = "contourpy" version = "1.3.3" @@ -434,10 +640,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/91/53255615acd2a1eaca307ede3c90eb550bae9c94581f8c00081b6b1c8f44/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57", size = 75987, upload-time = "2026-03-09T13:15:39.65Z" }, ] +[[package]] +name = "libdatachannel-py" +version = "2026.1.0.dev2" +source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" } +resolution-markers = [ + "sys_platform == 'win32'", +] +wheels = [ + { url = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/libdatachannel_py-2026.1.0.dev2-cp312-cp312-win_amd64.whl", hash = "sha256:9600b9eaa1e1ec27511522126a2d6886af5d0808a0f3f6dc3d63b7247dacba2e" }, +] + [[package]] name = "libdatachannel-py" version = "2026.1.0.dev2" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/46/2f/68e8306327ddef4b2133d2efb163cb05b319759ce8bd50b8b32dcd03dd95/libdatachannel_py-2026.1.0.dev2-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:6607fa1439e1b5bfceecd387c433470c9d45e439c3c06fa064f5c4669ad7e582", size = 1213155, upload-time = "2026-05-19T03:37:12.796Z" }, { url = "https://files.pythonhosted.org/packages/fc/e3/10aed36ffaf1744795322aae612db777991575b72a9f04e2c677c2c022bf/libdatachannel_py-2026.1.0.dev2-cp312-cp312-macosx_26_0_arm64.whl", hash = "sha256:a060b1250f57d1fccb36e3a6b36ac8f4fd34926a6b51c564e787e8b7206458aa", size = 1224706, upload-time = "2026-05-19T03:37:12.679Z" }, @@ -564,6 +784,7 @@ dependencies = [ requires-dist = [ { name = "cffi", marker = "extra == 'testing'" }, { name = "codespell", marker = "extra == 'testing'" }, + { name = "comma-deps-cppcheck", marker = "python_full_version >= '3.12' and extra == 'testing'" }, { name = "cpplint", marker = "extra == 'testing'" }, { name = "gcovr", marker = "extra == 'testing'" }, { name = "inputs", marker = "extra == 'examples'" }, @@ -582,25 +803,33 @@ requires-dist = [ provides-extras = ["testing", "examples"] [package.metadata.requires-dev] -testing = [ - { name = "comma-car-segments", url = "https://huggingface.co/datasets/commaai/commaCarSegments/resolve/main/dist/comma_car_segments-0.1.0-py3-none-any.whl" }, - { name = "cppcheck", git = "https://github.com/commaai/dependencies.git?subdirectory=cppcheck&rev=release-cppcheck" }, -] +testing = [{ name = "comma-car-segments", url = "https://huggingface.co/datasets/commaai/commaCarSegments/resolve/main/dist/comma_car_segments-0.1.0-py3-none-any.whl" }] [[package]] name = "openpilot" version = "0.1.0" source = { editable = "." } dependencies = [ - { name = "comma-deps-acados" }, - { name = "comma-deps-capnproto" }, - { name = "comma-deps-ffmpeg" }, - { name = "comma-deps-gcc-arm-none-eabi" }, - { name = "comma-deps-git-lfs" }, - { name = "comma-deps-json11" }, - { name = "comma-deps-raylib" }, - { name = "comma-deps-zeromq" }, - { name = "comma-deps-zstd" }, + { name = "comma-deps-acados", version = "0.2.2.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-acados", version = "0.2.2.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-capnproto", version = "1.0.1.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-capnproto", version = "1.0.1.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-eigen", version = "3.4.0.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-eigen", version = "3.4.0.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-ffmpeg", version = "7.1.0.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-ffmpeg", version = "7.1.0.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-gcc-arm-none-eabi", version = "13.2.1.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-gcc-arm-none-eabi", version = "13.2.1.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-git-lfs", version = "3.6.1.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-git-lfs", version = "3.6.1.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-json11", version = "20170411.0.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-json11", version = "20170411.0.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-raylib", version = "6.0.0.1.post101", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-raylib", version = "6.0.0.1.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-zeromq", version = "4.3.5.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-zeromq", version = "4.3.5.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-zstd", version = "1.5.6.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-zstd", version = "1.5.6.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, { name = "inputs" }, { name = "jeepney" }, { name = "numpy" }, @@ -618,6 +847,7 @@ dependencies = [ [package.optional-dependencies] submodules = [ + { name = "libdatachannel-py", version = "2026.1.0.dev2", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, { name = "msgq" }, { name = "opendbc" }, { name = "pandacan" }, @@ -632,10 +862,14 @@ testing = [ { name = "ty" }, ] tools = [ - { name = "comma-deps-bootstrap-icons" }, - { name = "comma-deps-imgui" }, - { name = "comma-deps-libusb" }, - { name = "comma-deps-ncurses" }, + { name = "comma-deps-bootstrap-icons", version = "1.10.5.0.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-bootstrap-icons", version = "1.10.5.0.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-imgui", version = "1.92.7.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-imgui", version = "1.92.7.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-libusb", version = "1.0.29.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-libusb", version = "1.0.29.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "comma-deps-ncurses", version = "6.5.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-ncurses", version = "6.5.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, { name = "matplotlib" }, ] @@ -647,22 +881,38 @@ standalone = [ [package.metadata] requires-dist = [ { name = "codespell", marker = "extra == 'testing'" }, - { name = "comma-deps-acados" }, - { name = "comma-deps-bootstrap-icons", marker = "extra == 'tools'" }, - { name = "comma-deps-capnproto" }, - { name = "comma-deps-ffmpeg" }, - { name = "comma-deps-gcc-arm-none-eabi" }, - { name = "comma-deps-git-lfs" }, - { name = "comma-deps-imgui", marker = "extra == 'tools'" }, - { name = "comma-deps-json11" }, - { name = "comma-deps-libusb", marker = "extra == 'tools'" }, - { name = "comma-deps-ncurses", marker = "extra == 'tools'" }, - { name = "comma-deps-raylib" }, - { name = "comma-deps-zeromq" }, - { name = "comma-deps-zstd" }, + { name = "comma-deps-acados", marker = "sys_platform != 'win32'" }, + { name = "comma-deps-acados", marker = "sys_platform == 'win32'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-bootstrap-icons", marker = "sys_platform == 'win32' and extra == 'tools'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-bootstrap-icons", marker = "sys_platform != 'win32' and extra == 'tools'" }, + { name = "comma-deps-capnproto", marker = "sys_platform != 'win32'" }, + { name = "comma-deps-capnproto", marker = "sys_platform == 'win32'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-eigen", marker = "sys_platform != 'win32'" }, + { name = "comma-deps-eigen", marker = "sys_platform == 'win32'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-ffmpeg", marker = "sys_platform != 'win32'" }, + { name = "comma-deps-ffmpeg", marker = "sys_platform == 'win32'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-gcc-arm-none-eabi", marker = "sys_platform != 'win32'" }, + { name = "comma-deps-gcc-arm-none-eabi", marker = "sys_platform == 'win32'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-git-lfs", marker = "sys_platform != 'win32'" }, + { name = "comma-deps-git-lfs", marker = "sys_platform == 'win32'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-imgui", marker = "sys_platform == 'win32' and extra == 'tools'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-imgui", marker = "sys_platform != 'win32' and extra == 'tools'" }, + { name = "comma-deps-json11", marker = "sys_platform != 'win32'" }, + { name = "comma-deps-json11", marker = "sys_platform == 'win32'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-libusb", marker = "sys_platform == 'win32' and extra == 'tools'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-libusb", marker = "sys_platform != 'win32' and extra == 'tools'" }, + { name = "comma-deps-ncurses", marker = "sys_platform == 'win32' and extra == 'tools'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-ncurses", marker = "sys_platform != 'win32' and extra == 'tools'" }, + { name = "comma-deps-raylib", marker = "sys_platform != 'win32'" }, + { name = "comma-deps-raylib", marker = "sys_platform == 'win32'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-zeromq", marker = "sys_platform != 'win32'" }, + { name = "comma-deps-zeromq", marker = "sys_platform == 'win32'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, + { name = "comma-deps-zstd", marker = "sys_platform != 'win32'" }, + { name = "comma-deps-zstd", marker = "sys_platform == 'win32'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, { name = "coverage", marker = "extra == 'testing'" }, { name = "inputs" }, { name = "jeepney" }, + { name = "libdatachannel-py", marker = "sys_platform == 'win32' and extra == 'submodules'", index = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, { name = "matplotlib", marker = "extra == 'tools'" }, { name = "msgq", marker = "extra == 'submodules'", editable = "msgq_repo" }, { name = "numpy", specifier = ">=2.0" }, @@ -712,9 +962,9 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "cffi", marker = "extra == 'dev'" }, - { name = "cppcheck", marker = "extra == 'dev'", git = "https://github.com/commaai/dependencies.git?subdirectory=cppcheck&rev=release-cppcheck" }, + { name = "comma-deps-cppcheck", marker = "python_full_version >= '3.12' and extra == 'dev'" }, + { name = "comma-deps-gcc-arm-none-eabi", marker = "python_full_version >= '3.12' and extra == 'dev'" }, { name = "flaky", marker = "extra == 'dev'" }, - { name = "gcc-arm-none-eabi", marker = "extra == 'dev'", git = "https://github.com/commaai/dependencies.git?subdirectory=gcc-arm-none-eabi&rev=release-gcc-arm-none-eabi" }, { name = "libusb-package" }, { name = "libusb1" }, { name = "opendbc", git = "https://github.com/commaai/opendbc.git?rev=master" }, @@ -856,7 +1106,8 @@ version = "0.0.1" source = { editable = "rednose_repo" } dependencies = [ { name = "cffi" }, - { name = "comma-deps-eigen" }, + { name = "comma-deps-eigen", version = "3.4.0.post98", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "comma-deps-eigen", version = "3.4.0.post115", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, { name = "cython" }, { name = "numpy" }, { name = "scons" }, @@ -1003,7 +1254,8 @@ name = "teleoprtc" version = "1.0.1" source = { editable = "teleoprtc_repo" } dependencies = [ - { name = "libdatachannel-py" }, + { name = "libdatachannel-py", version = "2026.1.0.dev2", source = { registry = "https://github.com/AmyJeanes/commaai-dependencies/releases/download/windows-post115/index.html" }, marker = "sys_platform == 'win32'" }, + { name = "libdatachannel-py", version = "2026.1.0.dev2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, ] [package.metadata]