Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
18 kB · 616 lines
Nix
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617{ lib, stdenv, runCommand, fetchurl, fetchFromGitHub, fetchPypi, fetchpatch2,
# Build time autoconf, automake, cmake, ensureNewerSourcesHook, fmt, git, libtool, makeWrapper, nasm, pkg-config, which, openssl,
# Tests nixosTests,
# Runtime dependencies arrow-cpp, babeltrace, # Note when trying to upgrade boost: # * When upgrading Ceph, it's recommended to check which boost version Ceph uses on Fedora, # and default to that. # * The version that Ceph downloads if `-DWITH_SYSTEM_BOOST:BOOL=ON` is not given # is declared in `cmake/modules/BuildBoost.cmake` line `set(boost_version ...)`. # # If you want to upgrade to boost >= 1.86, you need a Ceph version that # has this PR in: # https://github.com/ceph/ceph/pull/61312 boost183, bzip2, cryptsetup, cunit, e2fsprogs, doxygen, gperf, graphviz, gnugrep, gtest, icu, kmod, libcap, libcap_ng, libnbd, libnl, libxml2, lmdb, lttng-ust, lua, lvm2, lz4, oath-toolkit, openldap, parted, python311, # to get an idea which Python versions are supported by Ceph, see upstream `do_cmake.sh` (see `PYBUILD=` variable) rdkafka, rocksdb, snappy, openssh, sqlite, utf8proc, xfsprogs, zlib, zstd,
# Dependencies of overridden Python dependencies, hopefully we can remove these soon. rustPlatform,
# Optional Dependencies curl ? null, expat ? null, fuse ? null, libatomic_ops ? null, libedit ? null, libs3 ? null, yasm ? null,
# Mallocs gperftools ? null, jemalloc ? null,
# Crypto Dependencies cryptopp ? null, nspr ? null, nss ? null,
# Linux Only Dependencies linuxHeaders, util-linux, libuuid, udev, keyutils, rdma-core, rabbitmq-c, libaio ? null, libxfs ? null, liburing ? null, zfs ? null, ...}:
# We must have one crypto libraryassert cryptopp != null || (nss != null && nspr != null);
let shouldUsePkg = pkg: if pkg != null && lib.meta.availableOn stdenv.hostPlatform pkg then pkg else null;
optYasm = shouldUsePkg yasm; optExpat = shouldUsePkg expat; optCurl = shouldUsePkg curl; optFuse = shouldUsePkg fuse; optLibedit = shouldUsePkg libedit; optLibatomic_ops = shouldUsePkg libatomic_ops; optLibs3 = shouldUsePkg libs3;
optJemalloc = shouldUsePkg jemalloc; optGperftools = shouldUsePkg gperftools;
optCryptopp = shouldUsePkg cryptopp; optNss = shouldUsePkg nss; optNspr = shouldUsePkg nspr;
optLibaio = shouldUsePkg libaio; optLibxfs = shouldUsePkg libxfs; optZfs = shouldUsePkg zfs;
# Downgrade rocksdb, 7.10 breaks ceph rocksdb' = rocksdb.overrideAttrs { version = "7.9.2"; src = fetchFromGitHub { owner = "facebook"; repo = "rocksdb"; rev = "refs/tags/v7.9.2"; hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM="; }; };
hasRadosgw = optExpat != null && optCurl != null && optLibedit != null;
# Malloc implementation (can be jemalloc, tcmalloc or null) malloc = if optJemalloc != null then optJemalloc else optGperftools;
# We prefer nss over cryptopp cryptoStr = if optNss != null && optNspr != null then "nss" else if optCryptopp != null then "cryptopp" else "none";
cryptoLibsMap = { nss = [ optNss optNspr ]; cryptopp = [ optCryptopp ]; none = [ ]; };
getMeta = description: { homepage = "https://ceph.io/en/"; inherit description; license = with lib.licenses; [ lgpl21 gpl2Only bsd3 mit publicDomain ]; maintainers = with lib.maintainers; [ adev ak johanot krav nh2 benaryorg ]; platforms = [ "x86_64-linux" "aarch64-linux" ]; };
ceph-common = with python.pkgs; buildPythonPackage { pname = "ceph-common"; format = "setuptools"; inherit src version;
sourceRoot = "ceph-${version}/src/python-common";
propagatedBuildInputs = [ pyyaml ];
nativeCheckInputs = [ pytestCheckHook ];
disabledTests = [ # requires network access "test_valid_addr" ];
meta = getMeta "Ceph common module for code shared by manager modules"; };
# Watch out for python <> boost compatibility python = python311.override { self = python; packageOverrides = self: super: let bcryptOverrideVersion = "4.0.1"; in { # Ceph does not support the following yet: # * `bcrypt` > 4.0 # * `cryptography` > 40 # See: # * https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602 # * Upstream issue: https://tracker.ceph.com/issues/63529 # > Python Sub-Interpreter Model Used by ceph-mgr Incompatible With Python Modules Based on PyO3 # * Moved to issue: https://tracker.ceph.com/issues/64213 # > MGR modules incompatible with later PyO3 versions - PyO3 modules may only be initialized once per interpreter process
bcrypt = super.bcrypt.overridePythonAttrs (old: rec { pname = "bcrypt"; version = bcryptOverrideVersion; src = fetchPypi { inherit pname version; hash = "sha256-J9N1kDrIJhz+QEf2cJ0W99GNObHskqr3KvmJVSplDr0="; }; cargoRoot = "src/_bcrypt"; cargoDeps = rustPlatform.fetchCargoVendor { inherit pname version src cargoRoot ; hash = "sha256-8PyCgh/rUO8uynzGdgylAsb5k55dP9fCnf40UOTCR/M="; }; });
# We pin the older `cryptography` 40 here; # this also forces us to pin other packages, see below cryptography = self.callPackage ./old-python-packages/cryptography.nix { };
# This is the most recent version of `pyopenssl` that's still compatible with `cryptography` 40. # See https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602 # and https://github.com/pyca/pyopenssl/blob/d9752e44127ba36041b045417af8a0bf16ec4f1e/CHANGELOG.rst#2320-2023-05-30 pyopenssl = super.pyopenssl.overridePythonAttrs (old: rec { version = "23.1.1"; src = fetchPypi { pname = "pyOpenSSL"; inherit version; hash = "sha256-hBSYub7GFiOxtsR+u8AjZ8B9YODhlfGXkIF/EMyNsLc="; }; disabledTests = old.disabledTests or [ ] ++ [ "test_export_md5_digest" ]; disabledTestPaths = old.disabledTestPaths or [ ] ++ [ "tests/test_ssl.py" ]; propagatedBuildInputs = old.propagatedBuildInputs or [ ] ++ [ self.flaky ]; # hack: avoid building docs due to incompatibility with current sphinx nativeBuildInputs = [ openssl ]; # old.nativeBuildInputs but without sphinx* outputs = lib.filter (o: o != "doc") old.outputs; });
# This is the most recent version of `trustme` that's still compatible with `cryptography` 40. # See https://github.com/NixOS/nixpkgs/issues/359723 # and https://github.com/python-trio/trustme/commit/586f7759d5c27beb44da60615a71848eb2a5a490 trustme = self.callPackage ./old-python-packages/trustme.nix { };
fastapi = super.fastapi.overridePythonAttrs (old: { # Flaky test: # ResourceWarning: Unclosed <MemoryObjectSendStream> # Unclear whether it's flaky in general or only in this overridden package set. doCheck = false; });
# Ceph does not support `kubernetes` >= 19, see: # https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090 kubernetes = super.kubernetes.overridePythonAttrs (old: rec { version = "18.20.0"; src = fetchFromGitHub { owner = "kubernetes-client"; repo = "python"; rev = "v${version}"; sha256 = "1sawp62j7h0yksmg9jlv4ik9b9i1a1w9syywc9mv8x89wibf5ql1"; fetchSubmodules = true; }; });
}; };
boost' = boost183.override { enablePython = true; inherit python; };
# TODO: split this off in build and runtime environment ceph-python-env = python.withPackages ( ps: with ps; [ ceph-common
# build time cython_0
# debian/control bcrypt cherrypy influxdb jinja2 kubernetes natsort numpy pecan prettytable pyjwt pyopenssl python-dateutil pyyaml requests routes scikit-learn scipy setuptools sphinx virtualenv werkzeug
# src/cephadm/zipapp-reqs.txt markupsafe
# src/pybind/mgr/requirements-required.txt cryptography jsonpatch
# src/tools/cephfs/shell/setup.py cmd2 colorama ] ); inherit (ceph-python-env.python) sitePackages;
version = "19.2.3"; src = fetchurl { url = "https://download.ceph.com/tarballs/ceph-${version}.tar.gz"; hash = "sha256-zlgp28C81SZbaFJ4yvQk4ZgYz4K/aZqtcISTO8LscSU="; };inrec { ceph = stdenv.mkDerivation { pname = "ceph"; inherit src version;
patches = [ ./boost-1.85.patch
(fetchpatch2 { name = "ceph-boost-1.86-uuid.patch"; url = "https://github.com/ceph/ceph/commit/01306208eac492ee0e67bff143fc32d0551a2a6f.patch?full_index=1"; hash = "sha256-OnDrr72inzGXXYxPFQevsRZImSvI0uuqFHqtFU2dPQE="; })
# See: # * <https://github.com/boostorg/python/issues/394> # * <https://aur.archlinux.org/cgit/aur.git/commit/?h=ceph&id=8c5cc7d8deec002f7596b6d0860859a0a718f12b> # * <https://github.com/ceph/ceph/pull/60999> ./boost-1.86-PyModule.patch ];
nativeBuildInputs = [ autoconf # `autoreconf` is called, e.g. for `qatlib_ext` automake # `aclocal` is called, e.g. for `qatlib_ext` cmake fmt git makeWrapper libtool # used e.g. for `qatlib_ext` nasm pkg-config python python.pkgs.python # for the toPythonPath function python.pkgs.wrapPython which (ensureNewerSourcesHook { year = "1980"; }) # for building docs/man-pages presumably doxygen graphviz ];
buildInputs = cryptoLibsMap.${cryptoStr} ++ [ arrow-cpp babeltrace boost' bzip2 # Adding `ceph-python-env` here adds the env's `site-packages` to `PYTHONPATH` during the build. # This is important, otherwise the build system may not find the Python deps and then # silently skip installing ceph-volume and other Ceph python tools. ceph-python-env cryptsetup cunit e2fsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it gperf gtest icu libcap libnbd libnl libxml2 lmdb lttng-ust lua lvm2 # according to `debian/control` file, e.g. `pvs` command used by `src/ceph-volume/ceph_volume/api/lvm.py` lz4 malloc oath-toolkit openldap optLibatomic_ops optLibs3 optYasm parted # according to `debian/control` file, used by `src/ceph-volume/ceph_volume/util/disk.py` rdkafka rocksdb' snappy openssh # according to `debian/control` file, `ssh` command used by `cephadm` sqlite utf8proc xfsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it zlib zstd ] ++ lib.optionals stdenv.hostPlatform.isLinux [ keyutils libcap_ng liburing libuuid linuxHeaders optLibaio optLibxfs optZfs rabbitmq-c rdma-core udev util-linux ] ++ lib.optionals hasRadosgw [ optCurl optExpat optFuse optLibedit ];
# Picked up, amongst others, by `wrapPythonPrograms`. pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ];
# * `unset AS` because otherwise the Ceph CMake build errors with # configure: error: No modern nasm or yasm found as required. Nasm should be v2.11.01 or later (v2.13 for AVX512) and yasm should be 1.2.0 or later. # because the code at # https://github.com/intel/isa-l/blob/633add1b569fe927bace3960d7c84ed9c1b38bb9/configure.ac#L99-L191 # doesn't even consider using `nasm` or `yasm` but instead uses `$AS` # from `gcc-wrapper`. # (Ceph's error message is extra confusing, because it says # `No modern nasm or yasm found` when in fact it found e.g. `nasm` # but then uses `$AS` instead. # * replace /sbin and /bin based paths with direct nix store paths # * increase the `command` buffer size since 2 nix store paths cannot fit within 128 characters preConfigure = '' unset AS
substituteInPlace src/common/module.c \ --replace "char command[128];" "char command[256];" \ --replace "/sbin/modinfo" "${kmod}/bin/modinfo" \ --replace "/sbin/modprobe" "${kmod}/bin/modprobe" \ --replace "/bin/grep" "${gnugrep}/bin/grep"
# The install target needs to be in PYTHONPATH for "*.pth support" check to succeed export PYTHONPATH=$PYTHONPATH:$lib/${sitePackages}:$out/${sitePackages} patchShebangs src/ '';
cmakeFlags = [ "-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib"
"-DWITH_CEPHFS_SHELL:BOOL=ON" "-DWITH_SYSTEMD:BOOL=OFF" # `WITH_JAEGER` requires `thrift` as a depenedncy (fine), but the build fails with: # CMake Error at src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-Release.cmake:49 (message): # Command failed: 2 # # 'make' 'opentelemetry_trace' 'opentelemetry_exporter_jaeger_trace' # # See also # # /build/ceph-18.2.0/build/src/opentelemetry-cpp/src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-*.log # and that file contains: # /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc: In member function 'virtual void opentelemetry::v1::exporter::jaeger::TUDPTransport::close()': # /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc:71:7: error: '::close' has not been declared; did you mean 'pclose'? # 71 | ::THRIFT_CLOSESOCKET(socket_); # | ^~~~~~~~~~~~~~~~~~ # Looks like `close()` is somehow not included. # But the relevant code is already removed in `open-telemetry` 1.10: https://github.com/open-telemetry/opentelemetry-cpp/pull/2031 # So it's probably not worth trying to fix that for this Ceph version, # and instead just disable Ceph's Jaeger support. "-DWITH_JAEGER:BOOL=OFF" "-DWITH_TESTS:BOOL=OFF"
# Use our own libraries, where possible "-DWITH_SYSTEM_ARROW:BOOL=ON" # Only used if other options enable Arrow support. "-DWITH_SYSTEM_BOOST:BOOL=ON" "-DWITH_SYSTEM_GTEST:BOOL=ON" "-DWITH_SYSTEM_ROCKSDB:BOOL=ON" "-DWITH_SYSTEM_UTF8PROC:BOOL=ON" "-DWITH_SYSTEM_ZSTD:BOOL=ON"
# Use our own python libraries too, see: # https://github.com/NixOS/nixpkgs/pull/344993#issuecomment-2391046329 "-DCEPHADM_BUNDLED_DEPENDENCIES=none"
# TODO breaks with sandbox, tries to download stuff with npm "-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF" # WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now ''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}'' ] ++ lib.optional stdenv.hostPlatform.isLinux "-DWITH_SYSTEM_LIBURING=ON";
preBuild = # The legacy-option-headers target is not correctly empbedded in the build graph. # It also contains some internal race conditions that we work around by building with `-j 1`. # Upstream discussion for additional context at https://tracker.ceph.com/issues/63402. '' cmake --build . --target legacy-option-headers -j 1 '';
postFixup = '' wrapPythonPrograms wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})"
# Test that ceph-volume exists since the build system has a tendency to # silently drop it with misconfigurations. test -f $out/bin/ceph-volume '';
outputs = [ "out" "lib" "dev" "doc" "man" ];
doCheck = false; # uses pip to install things from the internet
# Takes 7+h to build with 2 cores. requiredSystemFeatures = [ "big-parallel" ];
meta = getMeta "Distributed storage system";
passthru = { inherit version; inherit python; # to be able to test our overridden packages above individually with `nix-build -A` tests = { inherit (nixosTests) ceph-multi-node ceph-single-node ceph-single-node-bluestore ceph-single-node-bluestore-dmcrypt ; }; }; };
ceph-client = runCommand "ceph-client-${version}" { meta = getMeta "Tools needed to mount Ceph's RADOS Block Devices/Cephfs"; } '' mkdir -p $out/{bin,etc,${sitePackages},share/bash-completion/completions} cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin cp -r ${ceph}/bin/rbd-replay* $out/bin cp -r ${ceph}/sbin/mount.ceph $out/bin cp -r ${ceph}/sbin/mount.fuse.ceph $out/bin ln -s bin $out/sbin cp -r ${ceph}/${sitePackages}/* $out/${sitePackages} cp -r ${ceph}/etc/bash_completion.d $out/share/bash-completion/completions # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths substituteInPlace $out/bin/ceph --replace ${ceph} $out substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out '';}