Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
4.9 kB · 165 lines
Python
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166
import jsonfrom pathlib import Pathimport multiprocessingimport subprocessimport sysimport tomlfrom urllib.parse import urlparseimport yaml
import dag
# This should match the behavior of the default unpackPhase.# See https://github.com/NixOS/nixpkgs/blob/59fa082abdbf462515facc8800d517f5728c909d/pkgs/stdenv/generic/setup.sh#L1044archive_extensions = [ # xz extensions ".tar.xz", ".tar.lzma", ".txz",
# *.tar or *.tar.* ".tar", ".tar.Z", ".tar.bz2", ".tar.gz",
# Other tar extensions ".tgz", ".tbz2", ".tbz",
".zip" ]
def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin): depends_on = set() if closure_dependencies_dag.has_node(uuid): depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids)
other_libs = extra_libs.get(uuid, [])
if is_darwin: fixup = f"""fixupPhase = let libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path)) [{" ".join(["uuid-" + x for x in depends_on])}]; in ''
''""" else: fixup = f"""fixupPhase = let libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path)) [{" ".join(["uuid-" + x for x in depends_on])}]; in '' find $out -type f -executable -exec \ patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \; find $out -type f -executable -exec \ patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \; ''"""
return f"""stdenv.mkDerivation {{ name = "{artifact_name}"; src = fetchurl {{ url = "{url}"; sha256 = "{sha256}"; }}; preUnpack = '' mkdir unpacked cd unpacked ''; sourceRoot = "."; dontConfigure = true; dontBuild = true; installPhase = "cp -r . $out"; {fixup}; }}"""
def get_plain_derivation(url, sha256): return f"""fetchurl {{ url = "{url}"; sha256 = "{sha256}"; }}"""
def process_item(args): item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin = args uuid, src = item lines = []
artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode()) if not artifacts: return f' uuid-{uuid} = {{}};\n'
lines.append(f' uuid-{uuid} = {{')
for artifact_name, details in artifacts.items(): if len(details["download"]) == 0: continue download = details["download"][0] url = download["url"] sha256 = download["sha256"]
git_tree_sha1 = details["git-tree-sha1"]
parsed_url = urlparse(url) if any(parsed_url.path.endswith(x) for x in archive_extensions): derivation = get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin) else: derivation = get_plain_derivation(url, sha256)
lines.append(f""" "{artifact_name}" = {{ sha1 = "{git_tree_sha1}"; path = {derivation}; }};\n""")
lines.append(' };\n')
return "\n".join(lines)
def main(): dependencies_path = Path(sys.argv[1]) closure_yaml_path = Path(sys.argv[2]) julia_path = Path(sys.argv[3]) extract_artifacts_script = Path(sys.argv[4]) extra_libs = json.loads(sys.argv[5]) is_darwin = json.loads(sys.argv[6]) out_path = Path(sys.argv[7])
with open(dependencies_path, "r") as f: dependencies = yaml.safe_load(f) dependency_uuids = list(dependencies.keys()) # Convert dict_keys to list
with open(closure_yaml_path, "r") as f: # Build up a map of UUID -> closure information closure_yaml_list = yaml.safe_load(f) or [] closure_yaml = {} for item in closure_yaml_list: closure_yaml[item["uuid"]] = item
# Build up a dependency graph of UUIDs closure_dependencies_dag = dag.DAG() for uuid, contents in closure_yaml.items(): if contents.get("depends_on"): closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())
with open(out_path, "w") as f: if is_darwin: f.write("{ lib, fetchurl, pkgs, stdenv }:\n\n") else: f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n")
f.write("rec {\n")
with multiprocessing.Pool(10) as pool: # Create args tuples for each item process_args = [ (item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin) for item in dependencies.items() ] for s in pool.map(process_item, process_args): f.write(s)
f.write(f"""}}\n""")
if __name__ == "__main__": main()