Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
8.7 kB · 293 lines
Nix
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294{ callPackage, fetchgit, fontconfig, git, lib, makeWrapper, python3, runCommand, writeTextFile,
# Artifacts dependencies fetchurl, glibc, pkgs, stdenv,
julia,
# Special registry which is equal to JuliaRegistries/General, but every Versions.toml # entry is augmented with a Nix sha256 hash augmentedRegistry ? callPackage ./registry.nix { },
# Other overridable arguments extraLibs ? [ ], juliaCpuTarget ? null, makeTransitiveDependenciesImportable ? false, # Used to support symbol indexing makeWrapperArgs ? "", packageOverrides ? { }, precompile ? true, setDefaultDepot ? true,}:
packageNames:
let util = callPackage ./util.nix { };
# Some Julia packages require access to Python. Provide a Nixpkgs version so it # doesn't try to install its own. pythonToUse = let extraPythonPackages = ( (callPackage ./extra-python-packages.nix { inherit python3; }).getExtraPythonPackages packageNames ); in ( if extraPythonPackages == [ ] then python3 else util.addPackagesToPython python3 (map (pkg: lib.getAttr pkg python3.pkgs) extraPythonPackages) );
# Start by wrapping Julia so it has access to Python and any other extra libs. # Also, prevent various packages (CondaPkg.jl, PythonCall.jl) from trying to do network calls. juliaWrapped = runCommand "julia-${julia.version}-wrapped" { nativeBuildInputs = [ makeWrapper ]; inherit makeWrapperArgs; } '' mkdir -p $out/bin makeWrapper ${julia}/bin/julia $out/bin/julia \ --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath extraLibs}" \ --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf \ --set PYTHONHOME "${pythonToUse}" \ --prefix PYTHONPATH : "${pythonToUse}/${pythonToUse.sitePackages}" \ --set PYTHON ${pythonToUse}/bin/python $makeWrapperArgs \ --set JULIA_CONDAPKG_OFFLINE yes \ --set JULIA_CONDAPKG_BACKEND Null \ --set JULIA_PYTHONCALL_EXE "@PyCall" '';
# If our closure ends up with certain packages, add others. packageImplications = { # Because we want to put PythonCall in PyCall mode so it doesn't try to download # Python packages PythonCall = [ "PyCall" ]; };
# Invoke Julia resolution logic to determine the full dependency closure packageOverridesRepoified = lib.mapAttrs util.repoifySimple packageOverrides; closureYaml = callPackage ./package-closure.nix { inherit augmentedRegistry julia packageNames packageImplications ; packageOverrides = packageOverridesRepoified; };
# Generate a Nix file consisting of a map from dependency UUID --> package info with fetchgit call: # { # "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" = { # src = fetchgit {...}; # name = "..."; # version = "..."; # treehash = "..."; # }; # ... # } dependencies = runCommand "julia-sources.nix" { buildInputs = [ (python3.withPackages ( ps: with ps; [ toml pyyaml ] )) git ]; } '' python ${./python}/sources_nix.py \ "${augmentedRegistry}" \ '${lib.generators.toJSON { } packageOverridesRepoified}' \ "${closureYaml}" \ "$out" '';
# Import the Nix file from the previous step (IFD) and turn each dependency repo into # a dummy Git repository, as Julia expects. Format the results as a YAML map from # dependency UUID -> Nix store location: # { # "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3":"/nix/store/...-NaNMath.jl-0877504", # ... # } # This is also the point where we apply the packageOverrides. dependencyUuidToInfo = import dependencies { inherit fetchgit; }; fillInOverrideSrc = uuid: info: if lib.hasAttr info.name packageOverrides then (info // { src = lib.getAttr info.name packageOverrides; }) else info; dependencyUuidToRepo = lib.mapAttrs util.repoifyInfo ( lib.mapAttrs fillInOverrideSrc dependencyUuidToInfo ); dependencyUuidToRepoYaml = writeTextFile { name = "dependency-uuid-to-repo.yml"; text = lib.generators.toYAML { } dependencyUuidToRepo; };
# Given the augmented registry, closure info yaml, and dependency path yaml, construct a complete # Julia registry containing all the necessary packages dependencyUuidToInfoYaml = writeTextFile { name = "dependency-uuid-to-info.yml"; text = lib.generators.toYAML { } dependencyUuidToInfo; }; fillInOverrideSrc' = uuid: info: if lib.hasAttr info.name packageOverridesRepoified then (info // { src = lib.getAttr info.name packageOverridesRepoified; }) else info; overridesOnly = lib.mapAttrs fillInOverrideSrc' ( lib.filterAttrs (uuid: info: info.src == null) dependencyUuidToInfo ); minimalRegistry = runCommand "minimal-julia-registry" { buildInputs = [ (python3.withPackages ( ps: with ps; [ toml pyyaml ] )) git ]; } '' python ${./python}/minimal_registry.py \ "${augmentedRegistry}" \ "${closureYaml}" \ '${lib.generators.toJSON { } overridesOnly}' \ "${dependencyUuidToRepoYaml}" \ "$out" '';
# Next, deal with artifacts. Scan each artifacts file individually and generate a Nix file that # produces the desired Overrides.toml. artifactsNix = runCommand "julia-artifacts.nix" { buildInputs = [ (python3.withPackages ( ps: with ps; [ toml pyyaml ] )) ]; } '' python ${./python}/extract_artifacts.py \ "${dependencyUuidToRepoYaml}" \ "${closureYaml}" \ "${juliaWrapped}/bin/julia" \ "${ if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl }" \ '${lib.generators.toJSON { } (import ./extra-libs.nix)}' \ '${lib.generators.toJSON { } (stdenv.hostPlatform.isDarwin)}' \ "$out" '';
# Import the artifacts Nix to build Overrides.toml (IFD) artifacts = import artifactsNix ( { inherit lib fetchurl pkgs stdenv ; } // lib.optionalAttrs (!stdenv.targetPlatform.isDarwin) { inherit glibc; } ); overridesJson = writeTextFile { name = "Overrides.json"; text = lib.generators.toJSON { } artifacts; }; overridesToml = runCommand "Overrides.toml" { buildInputs = [ (python3.withPackages (ps: with ps; [ toml ])) ]; } '' python ${./python}/format_overrides.py \ "${overridesJson}" \ "$out" '';
# Build a Julia project and depot. The project contains Project.toml/Manifest.toml, while the # depot contains package build products (including the precompiled libraries, if precompile=true) projectAndDepot = callPackage ./depot.nix { inherit closureYaml extraLibs juliaCpuTarget overridesToml packageImplications precompile ; julia = juliaWrapped; registry = minimalRegistry; packageNames = if makeTransitiveDependenciesImportable then lib.mapAttrsToList (uuid: info: info.name) dependencyUuidToInfo else packageNames; };
in
runCommand "julia-${julia.version}-env" { nativeBuildInputs = [ makeWrapper ];
passthru = { inherit julia; inherit juliaWrapped; inherit (julia) pname version meta;
# Expose the steps we used along the way in case the user wants to use them, for example to build # expressions and build them separately to avoid IFD. inherit dependencies; inherit closureYaml; inherit dependencyUuidToInfoYaml; inherit dependencyUuidToRepoYaml; inherit minimalRegistry; inherit artifactsNix; inherit overridesJson; inherit overridesToml; inherit projectAndDepot; }; } ( '' mkdir -p $out/bin makeWrapper ${juliaWrapped}/bin/julia $out/bin/julia \ --suffix JULIA_DEPOT_PATH : "${projectAndDepot}/depot" \ --set-default JULIA_PROJECT "${projectAndDepot}/project" \ --set-default JULIA_LOAD_PATH '@:${projectAndDepot}/project/Project.toml:@v#.#:@stdlib' '' + lib.optionalString setDefaultDepot '' sed -i '2 i\JULIA_DEPOT_PATH=''${JULIA_DEPOT_PATH-"$HOME/.julia"}' $out/bin/julia '' )