Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
5.4 kB · 215 lines
Nix
at gcc-offload
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216{ buildPythonPackage, cloudpickle, crcmod, cython, dill, fastavro, fasteners, fetchFromGitHub, fetchpatch, freezegun, grpcio, grpcio-tools, hdfs, httplib2, hypothesis, lib, mock, mypy-protobuf, numpy, objsize, orjson, pandas, parameterized, proto-plus, protobuf, psycopg2, pyarrow, pydot, pyhamcrest, pymongo, pytest-xdist, pytestCheckHook, python, python-dateutil, pytz, pyyaml, regex, requests, requests-mock, scikit-learn, setuptools, sqlalchemy, tenacity, testcontainers, typing-extensions, zstandard,}:
buildPythonPackage rec { pname = "apache-beam"; version = "2.59.0"; pyproject = true;
src = fetchFromGitHub { owner = "apache"; repo = "beam"; rev = "refs/tags/v${version}"; hash = "sha256-JeVYfXAx/GBGXQKAt6pSpnxH83oyeDylEY12EDzMxnw="; };
patches = [ (fetchpatch { # https://github.com/apache/beam/pull/24143 name = "fix-for-dill-0.3.6.patch"; url = "https://github.com/apache/beam/commit/7e014435b816015d21cc07f3f6c80809f3d8023d.patch"; hash = "sha256-iUmnzrItTFM98w3mpadzrmtI3t0fucpSujAg/6qxCGk="; stripLen = 2; }) ];
pythonRelaxDeps = [ # See https://github.com/NixOS/nixpkgs/issues/156957 "dill" "numpy" "pymongo"
# See https://github.com/NixOS/nixpkgs/issues/193613 "protobuf"
# As of apache-beam v2.45.0, the requirement is httplib2>=0.8,<0.21.0, but # the current (2023-02-08) nixpkgs's httplib2 version is 0.21.0. This can be # removed once beam is upgraded since the current requirement on master is # for httplib2>=0.8,<0.22.0. "httplib2"
# As of apache-beam v2.45.0, the requirement is pyarrow<10.0.0,>=0.15.1, but # the current (2023-02-22) nixpkgs's pyarrow version is 11.0.0. "pyarrow" ];
sourceRoot = "${src.name}/sdks/python";
nativeBuildInputs = [ cython grpcio-tools mypy-protobuf setuptools ];
propagatedBuildInputs = [ cloudpickle crcmod dill fastavro fasteners grpcio hdfs httplib2 numpy objsize orjson proto-plus protobuf pyarrow pydot pymongo python-dateutil pytz regex requests typing-extensions zstandard ];
enableParallelBuilding = true;
pythonImportsCheck = [ "apache_beam" ];
checkInputs = [ freezegun hypothesis mock pandas parameterized psycopg2 pyhamcrest pytestCheckHook pytest-xdist pyyaml requests-mock scikit-learn sqlalchemy tenacity testcontainers ];
# Make sure we're running the tests for the actually installed # package, so that cython's .so files are available. preCheck = "cd $out/${python.sitePackages}";
disabledTestPaths = [ # Fails with # _______ ERROR collecting apache_beam/io/external/xlang_jdbcio_it_test.py _______ # apache_beam/io/external/xlang_jdbcio_it_test.py:80: in <module> # class CrossLanguageJdbcIOTest(unittest.TestCase): # apache_beam/io/external/xlang_jdbcio_it_test.py:99: in CrossLanguageJdbcIOTest # container_init: Callable[[], Union[PostgresContainer, MySqlContainer]], # E NameError: name 'MySqlContainer' is not defined # "apache_beam/io/external/xlang_jdbcio_it_test.py"
# These tests depend on the availability of specific servers backends. "apache_beam/runners/portability/flink_runner_test.py" "apache_beam/runners/portability/samza_runner_test.py" "apache_beam/runners/portability/spark_runner_test.py"
# Fails starting from dill 0.3.6 because it tries to pickle pytest globals: # https://github.com/uqfoundation/dill/issues/482#issuecomment-1139017499. "apache_beam/transforms/window_test.py"
# See https://github.com/apache/beam/issues/25390. "apache_beam/coders/slow_coders_test.py" "apache_beam/dataframe/pandas_doctests_test.py" "apache_beam/typehints/typed_pipeline_test.py" "apache_beam/coders/fast_coders_test.py" "apache_beam/dataframe/schemas_test.py" ];
disabledTests = [ # The reasons of failures for these tests are unclear. # They reproduce in Docker with Ubuntu 22.04 # (= they're not `nixpkgs`-specific) but given the upstream uses # quite elaborate testing infra with containers and multiple # different runners - I don't expect them to help debugging these # when running via our (= custom from their PoV) testing infra. "test_with_main_session" # AssertionErrors "test_unified_repr" "testDictComprehension" "testDictComprehensionSimple" "testGenerator" "testGeneratorComprehension" "testListComprehension" "testNoneReturn" "testSet" "testTupleListComprehension" "test_newtype" "test_pardo_type_inference" "test_get_output_batch_type" "test_pformat_namedtuple_with_unnamed_fields" "test_row_coder_fail_early_bad_schema" # See https://github.com/apache/beam/issues/26004. "test_batch_encode_decode" ];
meta = with lib; { description = "Unified model for defining both batch and streaming data-parallel processing pipelines"; homepage = "https://beam.apache.org/"; license = licenses.asl20; maintainers = with maintainers; [ ndl ]; # https://github.com/apache/beam/issues/27221 broken = lib.versionAtLeast pandas.version "2"; };}