darwin: change the SDK pattern, update the SDKs, and update source releases

maintainers/team-list.nix

@@ -194,7 +194,10 @@ with lib.maintainers;
   };
 
   darwin = {
-    members = [ toonn ];
+    members = [
+      reckenrode
+      toonn
+    ];
     githubTeams = [ "darwin-maintainers" ];
     scope = "Maintain Darwin compatibility of packages and Darwin-only packages.";
     shortName = "Darwin";

pkgs/build-support/bintools-wrapper/add-darwin-ldflags-before.sh

@@ -79,3 +79,13 @@ if [ ! "$havePlatformVersionFlag" ]; then
         extraBefore+=(-@darwinPlatform@_version_min "${@darwinMinVersionVariable@_@suffixSalt@:-@darwinMinVersion@}")
     fi
 fi
+
+mangleVarSingle DEVELOPER_DIR ${role_suffixes[@]+"${role_suffixes[@]}"}
+
+# Allow wrapped bintools to do something useful when no `DEVELOPER_DIR` is set, which can happen when
+# the compiler is run outside of a stdenv or intentionally in an environment with no environment variables set.
+DEVELOPER_DIR=${DEVELOPER_DIR_@suffixSalt@:-@fallback_sdk@}
+
+# Darwin looks for frameworks in the SDK located at `DEVELOPER_DIR`.
+extraBefore+=("-F$DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks")
+extraBefore+=("-L$DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/lib")

pkgs/build-support/bintools-wrapper/default.nix

@@ -10,6 +10,7 @@
 , stdenvNoCC
 , runtimeShell
 , bintools ? null, libc ? null, coreutils ? null, gnugrep ? null
+, apple-sdk ? null
 , netbsd ? null
 , sharedLibraryLoader ?
   if libc == null then
@@ -107,7 +108,8 @@ let
   coreutils_bin = optionalString (!nativeTools) (getBin coreutils);
 
   # See description in cc-wrapper.
-  suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config;
+  suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config
+    + lib.optionalString (targetPlatform.isDarwin && targetPlatform.isStatic) "_static";
 
   # The dynamic linker has different names on different platforms. This is a
   # shell glob that ought to match it.
@@ -412,6 +414,9 @@ stdenvNoCC.mkDerivation {
     inherit dynamicLinker targetPrefix suffixSalt coreutils_bin;
     inherit bintools_bin libc_bin libc_dev libc_lib;
     default_hardening_flags_str = builtins.toString defaultHardeningFlags;
+  } // lib.optionalAttrs (apple-sdk != null && stdenvNoCC.targetPlatform.isDarwin) {
+    # Wrapped compilers should do something useful even when no SDK is provided at `DEVELOPER_DIR`.
+    fallback_sdk = apple-sdk.__spliced.buildTarget or apple-sdk;
   };
 
   meta =

pkgs/build-support/cc-wrapper/add-flags.sh

@@ -78,12 +78,28 @@ if [ -e @out@/nix-support/cc-cflags-before ]; then
     NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@="$(< @out@/nix-support/cc-cflags-before) $NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@"
 fi
 
-# Only add darwin min version flag if a default darwin min version is set,
+# Only add darwin min version flag and set up `DEVELOPER_DIR` if a default darwin min version is set,
 # which is a signal that we're targetting darwin.
 if [ "@darwinMinVersion@" ]; then
     mangleVarSingle @darwinMinVersionVariable@ ${role_suffixes[@]+"${role_suffixes[@]}"}
 
     NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@="-m@darwinPlatformForCC@-version-min=${@darwinMinVersionVariable@_@suffixSalt@:-@darwinMinVersion@} $NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@"
+
+    # `DEVELOPER_DIR` is used to dynamically locate libSystem (and the SDK frameworks) based on the SDK at that path.
+    mangleVarSingle DEVELOPER_DIR ${role_suffixes[@]+"${role_suffixes[@]}"}
+
+    # Allow wrapped compilers to do something useful when no `DEVELOPER_DIR` is set, which can happen when
+    # the compiler is run outside of a stdenv or intentionally in an environment with no environment variables set.
+    DEVELOPER_DIR=${DEVELOPER_DIR_@suffixSalt@:-@fallback_sdk@}
+
+    # xcbuild needs `SDKROOT` to be the name of the SDK, which it sets in its own wrapper,
+    # but compilers expect it to point to the absolute path.
+    SDKROOT="$DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"
+
+    # Set up various library paths since compilers may not support (or may have disabled) finding them in the sysroot.
+    NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@+=" -isysroot $SDKROOT"
+    NIX_CFLAGS_COMPILE_@suffixSalt@+=" -idirafter $SDKROOT/usr/include"
+    NIX_CFLAGS_COMPILE_@suffixSalt@+=" -iframework $SDKROOT/System/Library/Frameworks"
 fi
 
 # That way forked processes will not extend these environment variables again.

pkgs/build-support/cc-wrapper/default.nix

@@ -10,6 +10,7 @@
 , stdenvNoCC
 , runtimeShell
 , cc ? null, libc ? null, bintools, coreutils ? null
+, apple-sdk ? null
 , zlib ? null
 , nativeTools, noLibc ? false, nativeLibc, nativePrefix ? ""
 , propagateDoc ? cc != null && cc ? man
@@ -114,7 +115,8 @@ let
   # without interfering. For the moment, it is defined as the target triple,
   # adjusted to be a valid bash identifier. This should be considered an
   # unstable implementation detail, however.
-  suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config;
+  suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config
+    + lib.optionalString (targetPlatform.isDarwin && targetPlatform.isStatic) "_static";
 
   useGccForLibs = useCcForLibs
     && libcxx == null
@@ -749,6 +751,9 @@ stdenvNoCC.mkDerivation {
     inherit libc_bin libc_dev libc_lib;
     inherit darwinPlatformForCC darwinMinVersion darwinMinVersionVariable;
     default_hardening_flags_str = builtins.toString defaultHardeningFlags;
+  } // lib.optionalAttrs (apple-sdk != null && stdenvNoCC.targetPlatform.isDarwin) {
+    # Wrapped compilers should do something useful even when no SDK is provided at `DEVELOPER_DIR`.
+    fallback_sdk = apple-sdk.__spliced.buildTarget or apple-sdk;
   };
 
   meta =

pkgs/by-name/ap/apple-sdk/common/add-core-symbolication.nix

@@ -0,0 +1,51 @@
+{
+  lib,
+  fetchFromGitHub,
+  stdenvNoCC,
+}:
+
+let
+  CoreSymbolication = stdenvNoCC.mkDerivation (finalAttrs: {
+    pname = "CoreSymbolication";
+    version = "0-unstable-2018-06-17";
+
+    src = fetchFromGitHub {
+      repo = "CoreSymbolication";
+      owner = "matthewbauer";
+      rev = "24c87c23664b3ee05dc7a5a87d647ae476a680e4";
+      hash = "sha256-PzvLq94eNhP0+rLwGMKcMzxuD6MlrNI7iT/eV0obtSE=";
+    };
+
+    patches = [
+      # Add missing symbol definitions needed to build `zlog` in system_cmds.
+      # https://github.com/matthewbauer/CoreSymbolication/pull/2
+      ../patches/0001-Add-function-definitions-needed-to-build-zlog-in-sys.patch
+      ../patches/0002-Add-CF_EXPORT-To-const-symbols.patch
+    ];
+
+    dontBuild = true;
+
+    installPhase = ''
+      mkdir -p "$out/include"
+      cp *.h "$out/include"
+    '';
+
+    meta = {
+      description = "Reverse engineered headers for Apple's CoreSymbolication framework";
+      homepage = "https://github.com/matthewbauer/CoreSymbolication";
+      license = lib.licenses.mit;
+      maintainers = lib.teams.darwin.members;
+      platforms = lib.platforms.darwin;
+    };
+  });
+in
+self: super: {
+  buildPhase =
+    super.buildPhase or ""
+    + ''
+      mkdir -p System/Library/PrivateFrameworks/CoreSymbolication.framework/Versions/A/Headers
+      ln -s A System/Library/PrivateFrameworks/CoreSymbolication.framework/Versions/Current
+      ln -s Versions/Current/Headers System/Library/PrivateFrameworks/CoreSymbolication.framework/Headers
+      cp '${CoreSymbolication}/include/'*.h System/Library/PrivateFrameworks/CoreSymbolication.framework/Versions/A/Headers
+    '';
+}

pkgs/by-name/ap/apple-sdk/common/derivation-options.nix

@@ -0,0 +1,17 @@
+{ lib, config }:
+
+self: super: {
+  preBuild =
+    super.preBuild or ""
+    + ''
+      platformPath=$out/Platforms/MacOSX.platform
+      sdkpath=$platformPath/Developer/SDKs
+    '';
+
+  preInstall =
+    super.preInstall or ""
+    + ''
+      platformPath=$out/Platforms/MacOSX.platform
+      sdkpath=$platformPath/Developer/SDKs
+    '';
+}

pkgs/by-name/ap/apple-sdk/common/fetch-sdk.nix

@@ -0,0 +1,42 @@
+{
+  lib,
+  fetchurl,
+  cpio,
+  pbzx,
+}:
+
+{
+  url,
+  version,
+  hash,
+}:
+
+fetchurl {
+  pname = "macOS-SDK";
+  inherit version url hash;
+
+  recursiveHash = true;
+
+  nativeBuildInputs = [
+    cpio
+    pbzx
+  ];
+
+  postFetch = ''
+    renamed=$(mktemp -d)/sdk.xar
+    mv "$downloadedFile" "$renamed"
+    pbzx "$renamed" | cpio -idm
+
+    # SDKs are inconsistent about whether MacOSX.sdk or MacOSX<version>.sdk is a symlink.
+    src=Library/Developer/CommandLineTools/SDKs/MacOSX${lib.versions.majorMinor version}.sdk
+    if [ ! -d $src ]; then
+      src=Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
+    fi
+
+    # Remove unwanted binaries, man pages, and folders from the SDK.
+    rm -rf $src/usr/bin $src/usr/share $src/System/Library/Perl
+
+    mkdir -p "$out"
+    cp -rd $src/* "$out"
+  '';
+}

pkgs/by-name/ap/apple-sdk/common/passthru-private-frameworks.nix

@@ -0,0 +1,9 @@
+{ makeSetupHook, sdkVersion }:
+
+self: super: {
+  passthru = super.passthru or { } // {
+    privateFrameworksHook = makeSetupHook {
+      name = "apple-sdk-private-frameworks-hook";
+    } ../setup-hooks/add-private-frameworks.sh;
+  };
+}

pkgs/by-name/ap/apple-sdk/common/passthru-source-release-files.nix

@@ -0,0 +1,37 @@
+let
+  lockfile = builtins.fromJSON (builtins.readFile ../metadata/apple-oss-lockfile.json);
+in
+
+{
+  lib,
+  fetchFromGitHub,
+  stdenvNoCC,
+  sdkVersion,
+}:
+
+let
+  sdkinfo = lockfile.${sdkVersion};
+in
+self: super: {
+  passthru = super.passthru or { } // {
+    # Returns the raw source from apple-oss-distributions repo.
+    # This is mostly useful for copying private headers needed to build other source releases.
+    #
+    # Note: The source releases are mostly not used to build the SDK. Unless they can be used to build binaries,
+    # they’re not used.
+    sourceRelease =
+      name:
+      let
+        lockinfo = sdkinfo.${name};
+      in
+      fetchFromGitHub {
+        owner = "apple-oss-distributions";
+        repo = name;
+        rev = lockinfo.rev or "${name}-${lockinfo.version}";
+        inherit (lockinfo) hash;
+      }
+      // {
+        inherit (lockinfo) version;
+      };
+  };
+}

pkgs/development/tools/xcbuild/platforms.nix → pkgs/by-name/ap/apple-sdk/common/plists.nix

@@ -1,7 +1,10 @@
-{ stdenv, runCommand, lib, sdks, xcodePlatform, writeText }:
+{
+  lib,
+  stdenvNoCC,
+  xcodePlatform,
+}:
 
 let
-
   inherit (lib.generators) toPlist;
 
   Info = {
@@ -10,11 +13,7 @@ let
     Name = lib.toLower xcodePlatform;
   };
 
-  Version = {
-    ProjectName = "OSXPlatformSupport";
-  };
-
-  # These files are all based off of Xcode spec fies found in
+  # These files are all based off of Xcode spec files found in
   # /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/Library/Xcode/PrivatePlugIns/IDEOSXSupportCore.ideplugin/Contents/Resources.
 
   # Based off of the "MacOSX Architectures.xcspec" file. All i386 stuff
@@ -24,14 +23,20 @@ let
       Identifier = "Standard";
       Type = "Architecture";
       Name = "Standard Architectures (Apple Silicon, 64-bit Intel)";
-      RealArchitectures = [ "arm64" "x86_64" ];
+      RealArchitectures = [
+        "arm64"
+        "x86_64"
+      ];
       ArchitectureSetting = "ARCHS_STANDARD";
     }
     {
       Identifier = "Universal";
       Type = "Architecture";
       Name = "Universal (Apple Silicon, 64-bit Intel)";
-      RealArchitectures = [ "arm64" "x86_64" ];
+      RealArchitectures = [
+        "arm64"
+        "x86_64"
+      ];
       ArchitectureSetting = "ARCHS_STANDARD_32_64_BIT";
     }
     {
@@ -44,19 +49,25 @@ let
       Identifier = "Standard64bit";
       Type = "Architecture";
       Name = "Apple Silicon, 64-bit Intel";
-      RealArchitectures = [ "arm64" "x86_64" ];
+      RealArchitectures = [
+        "arm64"
+        "x86_64"
+      ];
       ArchitectureSetting = "ARCHS_STANDARD_64_BIT";
     }
     {
-      Identifier = if stdenv.hostPlatform.isAarch64 then "arm64" else "x86_64";
+      Identifier = stdenvNoCC.hostPlatform.darwinArch;
       Type = "Architecture";
       Name = "Apple Silicon or Intel 64-bit";
     }
     {
       Identifier = "Standard_Including_64_bit";
       Type = "Architecture";
       Name = "Standard Architectures (including 64-bit)";
-      RealArchitectures = [ "arm64" "x86_64" ];
+      RealArchitectures = [
+        "arm64"
+        "x86_64"
+      ];
       ArchitectureSetting = "ARCHS_STANDARD_INCLUDING_64_BIT";
     }
   ];
@@ -283,19 +294,14 @@ let
     }
   ];
 
+  ToolchainInfo = {
+    Identifier = "com.apple.dt.toolchain.XcodeDefault";
+  };
 in
-
-runCommand "Platforms" {} ''
-  platform=$out/${xcodePlatform}.platform
-
-  install -D ${writeText "Info.plist" (toPlist {} Info)} $platform/Info.plist
-  install -D ${writeText "version.plist" (toPlist {} Version)} $platform/version.plist
-  install -D ${writeText "Architectures.xcspec" (toPlist {} Architectures)} $platform/Developer/Library/Xcode/Specifications/Architectures.xcspec
-  install -D ${writeText "PackageTypes.xcspec" (toPlist {} PackageTypes)} $platform/Developer/Library/Xcode/Specifications/PackageTypes.xcspec
-  install -D ${writeText "ProductTypes.xcspec" (toPlist {} ProductTypes)} $platform/Developer/Library/Xcode/Specifications/ProductTypes.xcspec
-
-  ln -s $platform $platform/usr
-
-  mkdir -p $platform/Developer
-  ln -s ${sdks} $platform/Developer/SDKs
-''
+{
+  "Info.plist" = builtins.toFile "Info.plist" (toPlist { } Info);
+  "ToolchainInfo.plist" = builtins.toFile "ToolchainInfo.plist" (toPlist { } ToolchainInfo);
+  "Architectures.xcspec" = builtins.toFile "Architectures.xcspec" (toPlist { } Architectures);
+  "PackageTypes.xcspec" = builtins.toFile "PackageTypes.xcspec" (toPlist { } PackageTypes);
+  "ProductTypes.xcspec" = builtins.toFile "ProductTypes.xcspec" (toPlist { } ProductTypes);
+}