python312Packages.swh-storage: init at 3.1.0 (#394954)

pkgs/by-name/cm/cmph/package.nix

@@ -20,6 +20,7 @@ stdenv.mkDerivation (finalAttrs: {
       gpl2
       mpl11
     ];
+    mainProgram = "cmph";
     maintainers = with lib.maintainers; [ drupol ];
     platforms = lib.platforms.linux ++ lib.platforms.darwin;
   };

pkgs/by-name/sw/swh/package.nix

@@ -4,7 +4,10 @@
   withSwhPythonPackages ? [
     python3Packages.swh-auth
     python3Packages.swh-model
+    python3Packages.swh-objstorage
     python3Packages.swh-scanner
+    python3Packages.swh-scheduler
+    python3Packages.swh-storage
     python3Packages.swh-web-client
   ],
 }:

pkgs/development/python-modules/cassandra-driver/default.nix

@@ -21,6 +21,7 @@
   twisted,
   setuptools,
   distutils,
+  pythonAtLeast,
 }:
 
 buildPythonPackage rec {
@@ -114,6 +115,8 @@ buildPythonPackage rec {
   };
 
   meta = {
+    # cassandra/io/libevwrapper.c:668:10: error: implicit declaration of function ‘PyEval_ThreadsInitialized’ []
+    broken = pythonAtLeast "3.13";
     description = "Python client driver for Apache Cassandra";
     homepage = "http://datastax.github.io/python-driver";
     changelog = "https://github.com/datastax/python-driver/blob/${version}/CHANGELOG.rst";

pkgs/development/python-modules/plotille/default.nix

@@ -0,0 +1,66 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  fetchpatch,
+  poetry-core,
+  setuptools,
+  mock,
+  pendulum,
+  pytestCheckHook,
+  pytest-cov-stub,
+  pytest-mock,
+}:
+
+buildPythonPackage rec {
+  pname = "plotille";
+  version = "5.0.0";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "tammoippen";
+    repo = "plotille";
+    tag = "v${version}";
+    hash = "sha256-P2qwd935aaYgwLAKpTA2OAuohxVVzKwzYqjsuPSOSHs=";
+  };
+
+  patches = [
+    # To remove when PR https://github.com/tammoippen/plotille/pull/63 has landed
+    (fetchpatch {
+      name = "add-build-information";
+      url = "https://github.com/tammoippen/plotille/commit/db744e1fa9c141290966476ddf22a5e7d9a00c0a.patch";
+      hash = "sha256-8vBVKrcH7R1d9ol3D7RLVtAzZbpMsB9rA1KHD7t3Ydc=";
+    })
+  ];
+
+  postPatch = ''
+    substituteInPlace pyproject.toml \
+      --replace-fail poetry.masonry.api poetry.core.masonry.api \
+      --replace-fail "poetry>=" "poetry-core>="
+  '';
+
+  build-system = [
+    poetry-core
+    setuptools
+  ];
+
+  pythonImportsCheck = [
+    "plotille"
+  ];
+
+  nativeCheckInputs = [
+    mock
+    pendulum
+    pytestCheckHook
+    pytest-cov-stub
+    pytest-mock
+  ];
+
+  meta = {
+    changelog = "https://github.com/tammoippen/plotille/releases/tag/v${version}";
+    description = "Plot in the terminal using braille dots";
+    homepage = "https://github.com/tammoippen/plotille";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ drupol ];
+  };
+}

pkgs/development/python-modules/pytest-shared-session-scope/default.nix

@@ -0,0 +1,49 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  hatchling,
+  filelock,
+  pytest,
+  typing-extensions,
+  polars,
+  pytest-xdist,
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "pytest-shared-session-scope";
+  version = "0.4.0";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "StefanBRas";
+    repo = "pytest-shared-session-scope";
+    tag = "v${version}";
+    hash = "sha256-cG4RUwQwo7RyOQDCP54gGTLhnJtHTo5iQh8MjNRZ4HI=";
+  };
+
+  build-system = [ hatchling ];
+
+  dependencies = [
+    filelock
+    pytest
+    typing-extensions
+  ];
+
+  nativeCheckInputs = [
+    polars
+    pytest-xdist
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [ "pytest_shared_session_scope" ];
+
+  meta = {
+    changelog = "https://github.com/StefanBRas/pytest-shared-session-scope/blob/v${version}/CHANGELOG.md";
+    description = "Pytest session-scoped fixture that works with xdist";
+    homepage = "https://pypi.org/project/pytest-shared-session-scope/";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ drupol ];
+  };
+}

pkgs/development/python-modules/swh-journal/default.nix

@@ -0,0 +1,54 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitLab,
+  setuptools,
+  setuptools-scm,
+  confluent-kafka,
+  msgpack,
+  swh-core,
+  swh-model,
+  pytestCheckHook,
+  pytest-mock,
+}:
+
+buildPythonPackage rec {
+  pname = "swh-journal";
+  version = "2.0.0";
+  pyproject = true;
+
+  src = fetchFromGitLab {
+    domain = "gitlab.softwareheritage.org";
+    group = "swh";
+    owner = "devel";
+    repo = "swh-journal";
+    tag = "v${version}";
+    hash = "sha256-ycTB7hSjTerJOd+nEv8HbM82vPAO8P1+xooy0oN4eHw=";
+  };
+
+  build-system = [
+    setuptools
+    setuptools-scm
+  ];
+
+  dependencies = [
+    confluent-kafka
+    msgpack
+    swh-core
+    swh-model
+  ];
+
+  pythonImportsCheck = [ "swh.journal" ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+    pytest-mock
+  ];
+
+  meta = {
+    description = "Persistent logger of changes to the archive, with publish-subscribe support";
+    homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-journal";
+    license = lib.licenses.gpl3Only;
+    maintainers = with lib.maintainers; [ drupol ];
+  };
+}

pkgs/development/python-modules/swh-objstorage/default.nix

@@ -0,0 +1,111 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitLab,
+  setuptools,
+  setuptools-scm,
+  backports-entry-points-selectable,
+  cassandra-driver,
+  click,
+  deprecated,
+  flask,
+  iso8601,
+  msgpack,
+  mypy-extensions,
+  psycopg,
+  redis,
+  tenacity,
+  swh-core,
+  swh-model,
+  swh-perfecthash,
+  aiohttp,
+  azure-core,
+  azure-storage-blob,
+  fixtures,
+  libcloud,
+  postgresql,
+  postgresqlTestHook,
+  pytestCheckHook,
+  pytest-mock,
+  pytest-postgresql,
+  requests-mock,
+  requests-toolbelt,
+  systemd,
+  types-python-dateutil,
+  types-pyyaml,
+  types-requests,
+  util-linux,
+}:
+
+buildPythonPackage rec {
+  pname = "swh-objstorage";
+  version = "4.0.0";
+  pyproject = true;
+
+  src = fetchFromGitLab {
+    domain = "gitlab.softwareheritage.org";
+    group = "swh";
+    owner = "devel";
+    repo = "swh-objstorage";
+    tag = "v${version}";
+    hash = "sha256-c0ZH2PMT9DVnpTV5PDyX0Yw4iHiJSolEgq/bMXEwXG8=";
+  };
+
+  build-system = [
+    setuptools
+    setuptools-scm
+  ];
+
+  dependencies = [
+    backports-entry-points-selectable
+    cassandra-driver
+    click
+    deprecated
+    flask
+    iso8601
+    msgpack
+    mypy-extensions
+    psycopg
+    redis
+    tenacity
+    swh-core
+    swh-model
+    swh-perfecthash
+  ];
+
+  preCheck = ''
+    substituteInPlace swh/objstorage/backends/winery/roshard.py \
+      --replace-fail "/usr/bin/fallocate" "fallocate"
+  '';
+
+  pythonImportsCheck = [ "swh.objstorage" ];
+
+  pytestFlagsArray = [ "swh/objstorage/tests" ];
+
+  nativeCheckInputs = [
+    aiohttp
+    azure-core
+    azure-storage-blob
+    fixtures
+    libcloud
+    postgresql
+    postgresqlTestHook
+    pytestCheckHook
+    pytest-mock
+    pytest-postgresql
+    requests-mock
+    requests-toolbelt
+    systemd
+    types-python-dateutil
+    types-pyyaml
+    types-requests
+    util-linux
+  ] ++ psycopg.optional-dependencies.pool;
+
+  meta = {
+    description = "Content-addressable object storage for the Software Heritage project";
+    homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-objstorage";
+    license = lib.licenses.gpl3Only;
+    maintainers = with lib.maintainers; [ drupol ];
+  };
+}

pkgs/development/python-modules/swh-perfecthash/default.nix

@@ -0,0 +1,62 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitLab,
+  setuptools,
+  setuptools-scm,
+  cffi,
+  pytestCheckHook,
+  pytest-mock,
+  pkgs, # only for cmph
+}:
+
+buildPythonPackage rec {
+  pname = "swh-perfecthash";
+  version = "1.3.2";
+  pyproject = true;
+
+  src = fetchFromGitLab {
+    domain = "gitlab.softwareheritage.org";
+    group = "swh";
+    owner = "devel";
+    repo = "swh-perfecthash";
+    tag = "v${version}";
+    hash = "sha256-cG0h0lfSSooA7Mzrlsi5yIcbkbxQZ7mI5NtiB7D5Crs=";
+  };
+
+  build-system = [
+    setuptools
+    setuptools-scm
+  ];
+
+  dependencies = [
+    cffi
+    pkgs.cmph
+  ];
+
+  # The installed library clashes with the `swh` directory remaining in the source path.
+  # Usually, we get around this by `rm -rf` the python source files to ensure that the installed package is used.
+  # Here, we cannot do that as it would also remove the tests which are also in the `swh` directory.
+  preCheck = ''
+    rm -rf swh/perfecthash/*.py
+  '';
+
+  pythonImportsCheck = [ "swh.perfecthash" ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+    pytest-mock
+  ];
+
+  disabledTests = [
+    # Flake tests
+    "test_build_speed"
+  ];
+
+  meta = {
+    description = "Perfect hash table for software heritage object storage";
+    homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-perfecthash";
+    license = lib.licenses.gpl3Only;
+    maintainers = with lib.maintainers; [ drupol ];
+  };
+}

pkgs/development/python-modules/swh-scanner/default.nix

@@ -2,6 +2,7 @@
   lib,
   buildPythonPackage,
   fetchFromGitLab,
+  fetchpatch,
   setuptools,
   setuptools-scm,
   requests,
@@ -35,6 +36,15 @@ buildPythonPackage rec {
     hash = "sha256-baUUuYFapBD7iuDaDP8CSR9f4glVZcS5qBpZddVf7z8=";
   };
 
+  patches = [
+    # To be removed at the next release
+    # See https://gitlab.softwareheritage.org/swh/devel/swh-scanner/-/merge_requests/160
+    (fetchpatch {
+      url = "https://gitlab.softwareheritage.org/swh/devel/swh-scanner/-/commit/0eb273475826b0074844c7619b767c052562cfe4.patch";
+      hash = "sha256-i3hpaQJmHpIYgix+/npICQGtJ/IKVRXcCTm2O1VsR9M=";
+    })
+  ];
+
   build-system = [
     setuptools
     setuptools-scm
@@ -65,11 +75,6 @@ buildPythonPackage rec {
     types-requests
   ];
 
-  disabledTests = [
-    # AttributeError: 'called_once' is not a valid assertion. Use a spec for the mock if 'called_once' is meant to be an attribute.
-    "test_scan_api_url_option_success"
-  ];
-
   disabledTestPaths = [
     # pytestRemoveBytecodePhase fails with: "error (ignored): error: opening directory "/tmp/nix-build-python3.12-swh-scanner-0.8.3.drv-5/build/pytest-of-nixbld/pytest-0/test_randomdir_policy_info_cal0/big-directory/dir/dir/dir/ ......"
     "swh/scanner/tests/test_policy.py"

pkgs/development/python-modules/swh-scheduler/default.nix

@@ -0,0 +1,88 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitLab,
+  setuptools,
+  setuptools-scm,
+  celery,
+  flask,
+  humanize,
+  importlib-metadata,
+  pika,
+  psycopg,
+  tabulate,
+  swh-storage,
+  plotille,
+  postgresql,
+  postgresqlTestHook,
+  pytestCheckHook,
+  pytest-mock,
+  pytest-postgresql,
+  pytest-shared-session-scope,
+  pytest-xdist,
+  requests-mock,
+  simpy,
+  swh-journal,
+  types-python-dateutil,
+  types-pyyaml,
+  types-requests,
+}:
+
+buildPythonPackage rec {
+  pname = "swh-scheduler";
+  version = "3.1.0";
+  pyproject = true;
+
+  src = fetchFromGitLab {
+    domain = "gitlab.softwareheritage.org";
+    group = "swh";
+    owner = "devel";
+    repo = "swh-scheduler";
+    tag = "v${version}";
+    hash = "sha256-YpMHeZVHK8IPIiuBaPNR0D/yB9lIQ3DK7NEAiBmjWpA=";
+  };
+
+  build-system = [
+    setuptools
+    setuptools-scm
+  ];
+
+  dependencies = [
+    celery
+    flask
+    humanize
+    importlib-metadata
+    pika
+    psycopg
+    tabulate
+    swh-storage
+  ];
+
+  pythonImportsCheck = [ "swh.scheduler" ];
+
+  nativeCheckInputs = [
+    plotille
+    postgresql
+    postgresqlTestHook
+    pytestCheckHook
+    pytest-mock
+    pytest-postgresql
+    pytest-shared-session-scope
+    pytest-xdist
+    requests-mock
+    simpy
+    swh-journal
+    types-python-dateutil
+    types-pyyaml
+    types-requests
+  ];
+
+  disabledTests = [ "test_setup_log_handler_with_env_configuration" ];
+
+  meta = {
+    description = "Job scheduler for the Software Heritage project";
+    homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-scheduler";
+    license = lib.licenses.gpl3Only;
+    maintainers = with lib.maintainers; [ drupol ];
+  };
+}

pkgs/development/python-modules/swh-storage/default.nix

@@ -0,0 +1,100 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitLab,
+  setuptools,
+  setuptools-scm,
+  backports-entry-points-selectable,
+  cassandra-driver,
+  click,
+  deprecated,
+  flask,
+  iso8601,
+  mypy-extensions,
+  psycopg,
+  redis,
+  tenacity,
+  swh-core,
+  swh-model,
+  swh-objstorage,
+  postgresql,
+  postgresqlTestHook,
+  pytest-aiohttp,
+  pytest-mock,
+  pytest-postgresql,
+  pytest-shared-session-scope,
+  pytest-xdist,
+  pytestCheckHook,
+  swh-journal,
+}:
+
+buildPythonPackage rec {
+  pname = "swh-storage";
+  version = "3.1.0";
+  pyproject = true;
+
+  src = fetchFromGitLab {
+    domain = "gitlab.softwareheritage.org";
+    group = "swh";
+    owner = "devel";
+    repo = "swh-storage";
+    tag = "v${version}";
+    hash = "sha256-Bxwc8OccmqadLjHtmhToHBYHGkD7Fw3Cl3go9VLV/Bs=";
+  };
+
+  build-system = [
+    setuptools
+    setuptools-scm
+  ];
+
+  dependencies = [
+    backports-entry-points-selectable
+    cassandra-driver
+    click
+    deprecated
+    flask
+    iso8601
+    mypy-extensions
+    psycopg
+    redis
+    tenacity
+    swh-core
+    swh-model
+    swh-objstorage
+  ] ++ psycopg.optional-dependencies.pool;
+
+  pythonImportsCheck = [ "swh.storage" ];
+
+  nativeCheckInputs = [
+    postgresql
+    postgresqlTestHook
+    pytest-aiohttp
+    pytest-mock
+    pytest-postgresql
+    pytest-shared-session-scope
+    pytest-xdist
+    pytestCheckHook
+    swh-journal
+  ];
+
+  disabledTestPaths = [
+    # E       fixture 'redisdb' not found
+    "swh/storage/tests/test_replay.py"
+    # Unable to setup the local Cassandra database
+    "swh/storage/tests/test_cassandra.py"
+    "swh/storage/tests/test_cassandra_converters.py"
+    "swh/storage/tests/test_cassandra_diagram.py"
+    "swh/storage/tests/test_cassandra_migration.py"
+    "swh/storage/tests/test_cassandra_ttl.py"
+    "swh/storage/tests/test_cli_cassandra.py"
+    # Failing tests
+    "swh/storage/tests/test_cli_object_references.py"
+  ];
+
+  meta = {
+    description = "Abstraction layer over the archive, allowing to access all stored source code artifacts as well as their metadata";
+    homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-storage";
+    license = lib.licenses.gpl3Only;
+    maintainers = with lib.maintainers; [ drupol ];
+  };
+}

pkgs/top-level/python-packages.nix

@@ -11104,6 +11104,8 @@ self: super: with self; {
 
   ploomber-extension = callPackage ../development/python-modules/ploomber-extension { };
 
+  plotille = callPackage ../development/python-modules/plotille { };
+
   plotext = callPackage ../development/python-modules/plotext { };
 
   plotly = callPackage ../development/python-modules/plotly { };
@@ -13631,6 +13633,10 @@ self: super: with self; {
 
   pytest-services = callPackage ../development/python-modules/pytest-services { };
 
+  pytest-shared-session-scope =
+    callPackage ../development/python-modules/pytest-shared-session-scope
+      { };
+
   pytest-shutil = callPackage ../development/python-modules/pytest-shutil { };
 
   pytest-smtpd = callPackage ../development/python-modules/pytest-smtpd { };
@@ -16663,10 +16669,20 @@ self: super: with self; {
 
   swh-core = callPackage ../development/python-modules/swh-core { };
 
+  swh-journal = callPackage ../development/python-modules/swh-journal { };
+
   swh-model = callPackage ../development/python-modules/swh-model { };
 
+  swh-objstorage = callPackage ../development/python-modules/swh-objstorage { };
+
+  swh-perfecthash = callPackage ../development/python-modules/swh-perfecthash { };
+
   swh-scanner = callPackage ../development/python-modules/swh-scanner { };
 
+  swh-scheduler = callPackage ../development/python-modules/swh-scheduler { };
+
+  swh-storage = callPackage ../development/python-modules/swh-storage { };
+
   swh-web-client = callPackage ../development/python-modules/swh-web-client { };
 
   swift = callPackage ../development/python-modules/swift { };