From e2edfae5905cc12cc3682131386f8429a23e0f34 Mon Sep 17 00:00:00 2001 From: Steffeeen Date: Tue, 17 Mar 2026 16:55:34 +0100 Subject: [PATCH 1/2] Add SOURCEKIT_LSP_RUN_SOURCEKITD_IN_PROCESS environment variable This environment variable can be used to force SourceKit-LSP to run SourceKitD in-process on macOS. This can be useful for debugging. --- .../Environment Variables.md | 1 + Sources/ToolchainRegistry/Toolchain.swift | 35 ++++++++---- .../ToolchainRegistry/ToolchainRegistry.swift | 3 +- .../ToolchainRegistryTests.swift | 57 +++++++++++++++++++ 4 files changed, 83 insertions(+), 13 deletions(-) diff --git a/Contributor Documentation/Environment Variables.md b/Contributor Documentation/Environment Variables.md index b1d36259b..faa9b1dcf 100644 --- a/Contributor Documentation/Environment Variables.md +++ b/Contributor Documentation/Environment Variables.md @@ -12,6 +12,7 @@ The following environment variables can be used to control some behavior in Sour - `SOURCEKIT_LSP_LOG_LEVEL`: When using `NonDarwinLogger`, specify the level at which messages should be logged. Defaults to `debug` in debug build and `default` in release builds. Primarily used to increase the log level when running tests from a release build in Swift CI. To adjust the logging on user devices, use the [Configuration file](../Documentation/Configuration%20File.md). - `SOURCEKIT_LSP_LOG_PRIVACY_LEVEL`: When using `NonDarwinLogger`, specifies whether information that might contain sensitive information (essentially source code) should be logged. Defaults to `private` in debug build and `public` in release builds. Primarily used to log sensitive information when running tests from a release build in Swift CI. To adjust the logging on user devices, use the [Configuration file](../Documentation/Configuration%20File.md). +- `SOURCEKIT_LSP_RUN_SOURCEKITD_IN_PROCESS`: Force SourceKit-LSP to run SourceKit in process instead of using XPC on macOS. This is primarily intended for testing and debugging purposes. ## Testing - `SKIP_LONG_TESTS`: Skip tests that typically take more than 1s to execute. diff --git a/Sources/ToolchainRegistry/Toolchain.swift b/Sources/ToolchainRegistry/Toolchain.swift index 2c64b9569..81467e348 100644 --- a/Sources/ToolchainRegistry/Toolchain.swift +++ b/Sources/ToolchainRegistry/Toolchain.swift @@ -263,9 +263,14 @@ public final class Toolchain: Sendable { /// bin/clang /// /clangd /// /swiftc - /// lib/sourcekitd.framework/sourcekitd - /// /libsourcekitdInProc.{so,dylib} - /// /libIndexStore.{so,dylib} + /// lib/libIndexStore.{so,dylib} + /// ``` + /// and one of: + /// ``` + /// lib/sourcekitd.framework/sourcekitd (macOS) + /// lib/sourcekitdInProc.framework/sourcekitdInProc (macOS, if preferInProcessSourceKitD is true) + /// lib/libsourcekitdInProc.{so,dylib} (Linux or macOS, if the options above don't exist) + /// lib/sourcekitdInProc.dll (Windows) /// ``` /// /// The above directory layout can found relative to `path` in the following ways: @@ -275,7 +280,7 @@ public final class Toolchain: Sendable { /// /// If `path` contains an ".xctoolchain", we try to read an Info.plist file to provide the /// toolchain identifier, etc. Otherwise this information is derived from the path. - convenience package init?(_ path: URL) { + convenience package init?(_ path: URL, preferInProcessSourceKitD: Bool = false) { // Properties that need to be initialized let identifier: String let displayName: String @@ -360,16 +365,17 @@ public final class Toolchain: Sendable { dylibExtension = ".so" } - func findDylib(named name: String, searchFramework: Bool = false) -> URL? { + func findDylib(named name: String) -> URL? { let libSearchPath = libPath.appending(component: "lib\(name)\(dylibExtension)") if FileManager.default.isFile(at: libSearchPath) { return libSearchPath } + #if os(macOS) let frameworkPath = libPath.appending(components: "\(name).framework", name) - if searchFramework, FileManager.default.isFile(at: frameworkPath) { + if FileManager.default.isFile(at: frameworkPath) { return frameworkPath } - #if os(Windows) + #elseif os(Windows) let binSearchPath = binPath.appending(component: "\(name)\(dylibExtension)") if FileManager.default.isFile(at: binSearchPath) { return binSearchPath @@ -378,19 +384,24 @@ public final class Toolchain: Sendable { return nil } - if let sourcekitdPath = findDylib(named: "sourcekitd", searchFramework: true) - ?? findDylib(named: "sourcekitdInProc") - { + let sourcekitdPath: URL? = + if Platform.current != .darwin || preferInProcessSourceKitD { + findDylib(named: "sourcekitdInProc") + } else { + findDylib(named: "sourcekitd") ?? findDylib(named: "sourcekitdInProc") + } + + if let sourcekitdPath { sourcekitd = sourcekitdPath foundAny = true } - if let clientPluginPath = findDylib(named: "SwiftSourceKitClientPlugin", searchFramework: true) { + if let clientPluginPath = findDylib(named: "SwiftSourceKitClientPlugin") { sourceKitClientPlugin = clientPluginPath foundAny = true } - if let servicePluginPath = findDylib(named: "SwiftSourceKitPlugin", searchFramework: true) { + if let servicePluginPath = findDylib(named: "SwiftSourceKitPlugin") { sourceKitServicePlugin = servicePluginPath foundAny = true } diff --git a/Sources/ToolchainRegistry/ToolchainRegistry.swift b/Sources/ToolchainRegistry/ToolchainRegistry.swift index 48a7dbdf8..6f1c2d13c 100644 --- a/Sources/ToolchainRegistry/ToolchainRegistry.swift +++ b/Sources/ToolchainRegistry/ToolchainRegistry.swift @@ -170,6 +170,7 @@ package final actor ToolchainRegistry { xcodes: [URL] = [_currentXcodeDeveloperPath].compactMap({ $0 }), libraryDirectories: [URL] = FileManager.default.urls(for: .libraryDirectory, in: .allDomainsMask), pathEnvironmentVariables: [ProcessEnvironmentKey] = ["SOURCEKIT_PATH", "PATH"], + preferInProcessSourceKitD: Bool = ProcessEnv.block["SOURCEKIT_LSP_RUN_SOURCEKITD_IN_PROCESS"] != nil, darwinToolchainOverride: String? = ProcessEnv.block["TOOLCHAINS"] ) { // The paths at which we have found toolchains @@ -222,7 +223,7 @@ package final actor ToolchainRegistry { try toolchainAndReason.path.realpath } if let resolvedPath, - let toolchain = Toolchain(resolvedPath) + let toolchain = Toolchain(resolvedPath, preferInProcessSourceKitD: preferInProcessSourceKitD) { return (toolchain, toolchainAndReason.reason) } diff --git a/Tests/ToolchainRegistryTests/ToolchainRegistryTests.swift b/Tests/ToolchainRegistryTests/ToolchainRegistryTests.swift index 17ae4b6c2..fa84428b3 100644 --- a/Tests/ToolchainRegistryTests/ToolchainRegistryTests.swift +++ b/Tests/ToolchainRegistryTests/ToolchainRegistryTests.swift @@ -62,6 +62,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [xcodeDeveloper], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) @@ -105,6 +106,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [xcodeDeveloper], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) @@ -140,6 +142,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [xcodeDeveloper], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) @@ -176,6 +179,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [xcodeDeveloper], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) @@ -202,6 +206,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [], libraryDirectories: [libraryDir], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) assertEqual(await tr.toolchains.map(\.identifier), ["org.fake.global.A"]) @@ -232,6 +237,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) await assertEqual(trInstall.default?.identifier, "org.fake.explicit") @@ -268,6 +274,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [xcodeDeveloper], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) @@ -279,6 +286,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [xcodeDeveloper], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: "org.fake.global.A" ) await assertEqual(darwinToolchainOverrideRegistry.darwinToolchainIdentifier, "org.fake.global.A") @@ -381,6 +389,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) @@ -412,6 +421,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) @@ -575,6 +585,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) await assertNil(trEmpty.default) @@ -585,6 +596,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) await assertEqual(tr1.default?.path, binPath) @@ -596,6 +608,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) await assertNil(tr2.default) @@ -617,6 +630,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) await assertEqual(tr.toolchains.count, 2) @@ -642,6 +656,7 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { xcodes: [], libraryDirectories: [], pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, darwinToolchainOverride: nil ) await assertEqual(tr.toolchains.count, 2) @@ -652,6 +667,42 @@ final class ToolchainRegistryTests: SourceKitLSPTestCase { } } + func testSourceKitInProcessEnvVar() async throws { + try await withTestScratchDir { tempDir in + let bin = tempDir.appending(components: "t1", "bin", directoryHint: .isDirectory) + try makeToolchain(binPath: bin, sourcekitd: true, sourcekitdInProc: true) + + let libPath = bin.deletingLastPathComponent().appending(component: "lib") + + let tr1 = ToolchainRegistry( + installPath: bin, + environmentVariables: [], + xcodes: [], + libraryDirectories: [], + pathEnvironmentVariables: [], + preferInProcessSourceKitD: false, + darwinToolchainOverride: nil + ) + + await assertEqual(tr1.default?.sourcekitd, libPath.appending(components: "sourcekitd.framework", "sourcekitd")) + + let tr2 = ToolchainRegistry( + installPath: bin, + environmentVariables: [], + xcodes: [], + libraryDirectories: [], + pathEnvironmentVariables: [], + preferInProcessSourceKitD: true, + darwinToolchainOverride: nil + ) + + await assertEqual( + tr2.default?.sourcekitd, + libPath.appending(components: "sourcekitdInProc.framework", "sourcekitdInProc") + ) + } + } + func testSupersetToolchains() async throws { try await withTestScratchDir { tempDir in let usrLocal = tempDir.appending(components: "usr", "local") @@ -768,6 +819,12 @@ private func makeToolchain( if sourcekitdInProc { #if os(Windows) try Data().write(to: binPath.appending(component: "sourcekitdInProc\(dylibSuffix)")) + #elseif os(macOS) + try FileManager.default.createDirectory( + at: libPath.appending(component: "sourcekitdInProc.framework"), + withIntermediateDirectories: true + ) + try Data().write(to: libPath.appending(components: "sourcekitdInProc.framework", "sourcekitdInProc")) #else try Data().write(to: libPath.appending(component: "libsourcekitdInProc\(dylibSuffix)")) #endif From 76f76bc89c5d89d23e3abfa4138847b4b6d03cb7 Mon Sep 17 00:00:00 2001 From: Steffeeen Date: Tue, 24 Mar 2026 13:21:37 +0100 Subject: [PATCH 2/2] [ContributorDoc] Add detailed docs for using a local build of SourceKit --- CONTRIBUTING.md | 2 + Contributor Documentation/README.md | 1 + .../Using a Local Build of SourceKit.md | 79 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 Contributor Documentation/Using a Local Build of SourceKit.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cf17f3ff7..e2a890207 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,6 +92,8 @@ When SourceKit-LSP is installed as part of a toolchain, it finds the Swift versi To adjust the toolchain that should be used by SourceKit-LSP (eg. because you want to use new `sourcekitd` features that are only available in a Swift open source toolchain snapshot but not your default toolchain), set the `SOURCEKIT_TOOLCHAIN_PATH` environment variable to your toolchain when running SourceKit-LSP. +For a full walkthrough of building and using a local SourceKit toolchain, see [Using a Local Build of SourceKit](Contributor%20Documentation/Using%20a%20Local%20Build%20of%20SourceKit.md). + ## Logging SourceKit-LSP has extensive logging to the system log on macOS and to `~/.sourcekit-lsp/logs/` or stderr on other platforms. diff --git a/Contributor Documentation/README.md b/Contributor Documentation/README.md index 00f423b3d..ee3e95fcc 100644 --- a/Contributor Documentation/README.md +++ b/Contributor Documentation/README.md @@ -15,3 +15,4 @@ The following documentation documents are primarily intended for developers of S - [Overview](Overview.md) - [Swift Version](Swift%20Version.md) - [Testing](Testing.md) +- [Using a Local Build of SourceKit](Using%20a%20Local%20Build%20of%20SourceKit.md) diff --git a/Contributor Documentation/Using a Local Build of SourceKit.md b/Contributor Documentation/Using a Local Build of SourceKit.md new file mode 100644 index 000000000..3b8e06b10 --- /dev/null +++ b/Contributor Documentation/Using a Local Build of SourceKit.md @@ -0,0 +1,79 @@ +# Using a Local Build of SourceKit + +This guide explains how to run SourceKit-LSP against a local build of SourceKit from the Swift monorepo. + +Examples in this document use the checkout layout from Swift's [How to Set Up an Edit-Build-Test-Debug Loop](https://github.com/swiftlang/swift/blob/main/docs/HowToGuides/GettingStarted.md), ie.`swift-project/swift` and `swift-project/sourcekit-lsp` as sibling directories. +This layout is only for convenience. +`sourcekit-lsp` and `swift` can live in different directories as long as you pass the correct absolute paths. + +## 1. Build Swift + +Follow the [How to Set Up an Edit-Build-Test-Debug Loop](https://github.com/swiftlang/swift/blob/main/docs/HowToGuides/GettingStarted.md) guide to clone Swift and get the required dependencies. Then, add the following additional flags to the `build-script` invocation: `--swift-testing`, `--swift-testing-macros`, `--llbuild`, `--swiftpm` and `--install-all`. +These flags are necessary to build the components of Swift that SourceKit-LSP depends on, and to ensure that a toolchain with all the necessary components is created. +When using the `build-script` command from the guide the entire command becomes: + +- macOS: + + ```sh + utils/build-script --skip-build-benchmarks \ + --swift-darwin-supported-archs "$(uname -m)" \ + --release-debuginfo --swift-disable-dead-stripping \ + --bootstrapping=hosttools \ + --swift-testing \ + --swift-testing-macros \ + --llbuild \ + --swiftpm \ + --install-all + ``` + +- Linux: + + ```sh + utils/build-script --release-debuginfo \ + --swift-testing \ + --swift-testing-macros \ + --llbuild \ + --swiftpm \ + --install-all + ``` + +## 2. Point SourceKit-LSP to that toolchain + +Set the `SOURCEKIT_TOOLCHAIN_PATH` environment variable when running +tests or launching SourceKit-LSP to point to the `.xctoolchain` directory of the locally built toolchain. +The path should look something like this: `.../swift-project/build/Ninja-RelWithDebInfoAssert/toolchain-macosx-arm64/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain`. +You may need to adjust the path based on the build configuration and platform. + +Example (assuming `sourcekit-lsp` and `swift` are sibling directories): + +```bash +SOURCEKIT_TOOLCHAIN_PATH="$PWD/../build/Ninja-RelWithDebInfoAssert/toolchain-macosx-arm64/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain" \ +swift test +``` + +(you may need to adjust the path based on the build configuration and platform) + +## 3. Making Changes to SourceKit + +When making changes to SourceKit, you can iterate faster by only rebuilding SourceKit and its dependencies instead of rebuilding the entire Swift toolchain. +You need to ensure that the toolchain gets updated with the new build artifacts. +This can be done by running the following command from the `build/Ninja-RelWithDebInfoAssert/swift-macosx-arm64` directory after building Swift with the `build-script` command from step 1: + +```bash +DESTDIR=../toolchain-macosx-arm64 ninja sourcekitd sourcekit-inproc install-sourcekit-xpc-service install-sourcekit-inproc +``` + +(you may need to adjust the path based on the build configuration and platform) + +## SourceKit modes on macOS + +Normally, SourceKit-LSP uses XPC on macOS to communicate with SourceKit, while on Linux SourceKit runs in the same process as SourceKit-LSP. +However, sometimes it may be desirable to run SourceKit in-process on macOS as well. +Running SourceKit in-process can be useful when using the `SOURCEKIT_LOGGING` environment variable, as the output will include the log messages from SourceKit itself instead of just the messages from the SourceKit wrapper in SourceKit-LSP. +Additionally, running SourceKit in-process can be useful for profiling as the SourceKit symbols will be directly visible in the profile. + +To force SourceKit-LSP to run SourceKit in process, set the `SOURCEKIT_LSP_RUN_SOURCEKITD_IN_PROCESS` environment variable. + +```bash +SOURCEKIT_LSP_RUN_SOURCEKITD_IN_PROCESS=1 swift test +```