manticoresearch: 6.2.12 -> 15.1.0, add columnar (#451922)

pkgs/by-name/ma/manticoresearch/cctz-cmake-policy.patch

@@ -0,0 +1,9 @@
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -1,1 +1,2 @@
+-cmake_minimum_required(VERSION 2.8.12)
++# Fix the build with CMake 4
++cmake_minimum_required(VERSION 3.5)
+
+ if (POLICY CMP0025)
+   cmake_policy(SET CMP0025 NEW)

pkgs/by-name/ma/manticoresearch/columnar.nix

@@ -0,0 +1,172 @@
+{
+  cmake,
+  fetchFromGitHub,
+  fetchzip,
+  lib,
+  pkg-config,
+  replaceVars,
+  simde,
+  stdenv,
+}:
+
+let
+  # Manticore fetches columnar from a tag that indicates the versions of libraries that columnar includes.
+  # (See NEED_COLUMNAR_API/NEED_SECONDARY_API/NEED_KNN_API in Manticore's cmake/GetColumnar.cmake)
+  # So this tag indicates columnar v27, secondary v18, and knn v9.
+  libraryVersion = "c27-s18-k9";
+  # If you inspect the revision that tag points to, there's also hopefully a more normal version tag too
+  numericVersion = "9.0.0";
+  columnarSource = fetchFromGitHub {
+    owner = "manticoresoftware";
+    repo = "columnar";
+    tag = libraryVersion;
+    hash = "sha256-26zSdLNJkC7zYHT8sDq/2mZPRfdc9db1vIIrkaiPqFM=";
+  };
+  hnswlib = stdenv.mkDerivation (finalAttrs: {
+    pname = "hnswlib";
+    version = "d7bb3bbd59220f62203012b2f649aa537cc97cdc"; # see HNSW_GITHUB in columnar's cmake/GetHNSW.cmake
+    src = fetchFromGitHub {
+      owner = "manticoresoftware";
+      repo = "hnswlib";
+      rev = "d7bb3bbd59220f62203012b2f649aa537cc97cdc";
+      hash = "sha256-SDYf8J4auzqGBHqFMDFR6TVyNthCA4q6J4glcIssAJU=";
+    };
+    nativeBuildInputs = [
+      cmake
+      pkg-config
+    ];
+    meta = {
+      description = "Header-only C++/python library for fast approximate nearest neighbors";
+      homepage = "https://github.com/nmslib/hnswlib";
+      license = lib.licenses.asl20;
+      platforms = lib.platforms.all;
+    };
+  });
+  pgm-index = stdenv.mkDerivation (finalAttrs: {
+    pname = "pgm-index";
+    version = "pgm_2022_08_02"; # see PGM_GITHUB in columnar's cmake/GetPGM.cmake
+    src = fetchFromGitHub {
+      owner = "manticoresoftware";
+      repo = "PGM-index";
+      rev = "5a5a763c7c07e56f56fd33137dcee9f1b3c3b640";
+      hash = "sha256-p8CjY9r1TEi91HBac9rZ39Yae2XuNh2yQWbo65n90jU=";
+    };
+    nativeBuildInputs = [
+      cmake
+      pkg-config
+    ];
+    # Use the custom CMakeLists.txt from columnar
+    postPatch = ''
+      cp ${columnarSource}/pgm/CMakeLists.txt CMakeLists.txt
+    '';
+    meta = {
+      description = "Piecewise Geometric Model index - a fast and space-efficient data structure";
+      homepage = "https://github.com/gvinciguerra/PGM-index";
+      license = lib.licenses.asl20;
+      platforms = lib.platforms.all;
+    };
+  });
+  streamvbyte = stdenv.mkDerivation (finalAttrs: rec {
+    pname = "streamvbyte";
+    version = "efdd9dace81a4a8f844267631879b500c6d913cf"; # see SVB_GITHUB in columnar's cmake/GetStreamvbyte.cmake
+    src = fetchFromGitHub {
+      owner = "manticoresoftware";
+      repo = "streamvbyte";
+      rev = version;
+      hash = "sha256-a9E1aWBY/P7wI+kgHqhEiD3THctFfeFcy658RcNpHfQ=";
+    };
+    nativeBuildInputs = [
+      cmake
+      pkg-config
+    ];
+    # Enable SSE4.1 on x86_64 to avoid GCC 14 target attribute issues
+    env.NIX_CFLAGS_COMPILE = lib.optionalString stdenv.hostPlatform.isx86_64 "-msse4.1";
+    # Use the custom CMakeLists.txt from columnar
+    postPatch = ''
+      cp ${columnarSource}/streamvbyte/CMakeLists.txt CMakeLists.txt
+    '';
+    meta = {
+      description = "Fast integer compression library using StreamVByte encoding";
+      homepage = "https://github.com/manticoresoftware/streamvbyte/";
+      license = lib.licenses.asl20;
+      platforms = lib.platforms.all;
+    };
+  });
+  fastpfor = stdenv.mkDerivation (finalAttrs: {
+    pname = "fastpfor";
+    version = "simde"; # see FP_GITHUB in columnar's cmake/GetFastPFor.cmake
+    src = fetchFromGitHub {
+      owner = "manticoresoftware";
+      repo = "FastPFor";
+      rev = "5f6b7df8f0dba402bb616a81ca510bfeb04c42ce";
+      hash = "sha256-O9nqgsZKT+q9dHdh2RahF1U8KXCmDp0ld7hNvVREM8k=";
+    };
+    nativeBuildInputs = [
+      cmake
+      pkg-config
+    ];
+    buildInputs = [ simde ];
+    # Enable SSE3/SSSE3/SSE4.1 on x86_64 to avoid GCC 14 target attribute issues
+    # (Columnar's custom libfastpfor/CMakeLists.txt comments these out. I think get away with it because they just build with clang?)
+    env.NIX_CFLAGS_COMPILE = lib.optionalString stdenv.hostPlatform.isx86_64 "-msse3 -mssse3 -msse4.1";
+
+    # Columnar uses its own version of CMakeLists.txt to compile libfastpfor
+    prePatch = ''
+      cp ${columnarSource}/libfastpfor/CMakeLists.txt CMakeLists.txt
+    '';
+    patches = [ ./fastpfor-cmake-policy.patch ];
+    postPatch = ''
+      # Use nixpkgs simde rather than FetchContent-ing from github
+      sed -i '/Add simde for arm/,/FetchContent_GetProperties ( simde )/c\
+        message(STATUS "Using nixpkgs simde for arm")\n\
+        set(simde_SOURCE_DIR "${simde.src}")\n\
+        set(simde_POPULATED 1)\n\
+      ' CMakeLists.txt
+    '';
+    meta = {
+      description = "Fast integer compression";
+      homepage = "https://github.com/manticoresoftware/FastPFor/";
+      license = lib.licenses.asl20;
+      platforms = lib.platforms.all;
+    };
+  });
+in
+stdenv.mkDerivation (finalAttrs: {
+  pname = "manticore-columnar";
+  version = numericVersion;
+
+  outputs = [
+    "out"
+    "dev"
+  ];
+
+  src = columnarSource;
+  nativeBuildInputs = [
+    cmake
+  ];
+  buildInputs = [
+    fastpfor
+    hnswlib
+    pgm-index
+    streamvbyte
+  ];
+
+  patches = [
+    (replaceVars ./columnar.patch {
+      version = numericVersion;
+    })
+  ];
+  cmakeFlags = [
+    "-DSKIP_KNN=ON" # Skip KNN & embeddings for now
+  ];
+  postPatch = ''
+    # I've failed to get avx compiled, skip it for now.
+    sed -i '/set (ADD_AVX_BUILDS 1)/d' CMakeLists.txt
+  '';
+
+  meta = with lib; {
+    description = "Manticore Columnar Library - columnar storage and secondary indexes library for Manticore Search";
+    homepage = "https://github.com/manticoresoftware/columnar";
+    license = licenses.asl20;
+  };
+})

pkgs/by-name/ma/manticoresearch/columnar.patch

@@ -0,0 +1,33 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 3bfbc5ef9043..2a25188f8f92 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -26,7 +26,7 @@ set ( CMAKE_INTERPROCEDURAL_OPTIMIZATION $ENV{CMAKE_INTERPROCEDURAL_OPTIMIZATION
+ set ( _CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE} )
+ 
+ # The below version is used for versioning of the package and is replaced by the version from the pack workflow
+-project ( columnar VERSION 0.0.0 )
++project ( columnar VERSION @version@ )
+ 
+ # sometimes CMAKE_BUILD_TYPE became set after PROJECT statement, undo it.
+ if (NOT _CMAKE_BUILD_TYPE AND CMAKE_BUILD_TYPE)
+@@ -108,7 +108,7 @@ set (_includes include/manticore-columnar-api)
+ target_include_directories ( columnar_api INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>$<INSTALL_INTERFACE:${_includes}> )
+ 
+ # install columnar and secondary API (only headers, no packaging, don't build anything at all)
+-if (API_ONLY)
++# if (API_ONLY)
+ 	install ( TARGETS columnar_api EXPORT apiexport )
+ 	install ( TARGETS secondary_api EXPORT apiexport )
+ 	install ( TARGETS knn_api EXPORT apiexport )
+@@ -147,8 +147,8 @@ include(\"\${CMAKE_CURRENT_LIST_DIR}/columnar-targets.cmake\")" )
+ 	install ( FILES "${CMAKE_CURRENT_BINARY_DIR}/columnar-config-version.cmake" DESTINATION "${API_CMAKE_DIR}" )
+ 
+ 	# finish configuration, as it is API_ONLY
+-	return()
+-endif ()
++	# return()
++# endif ()
+ 
+ # here we came in case of full build (i.e. columnar and/or secondary and/or knn)
+ 

pkgs/by-name/ma/manticoresearch/fastpfor-cmake-policy.patch

@@ -0,0 +1,14 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index efedf74..effa1ec 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -4,7 +4,8 @@
+ # Copyright (c) 2012 Louis Dionne
+ # Copyright (c) 2021-2022 Manticore Software LTD
+ #
+-cmake_minimum_required ( VERSION 3.17 )
++# Fix build with CMake 4
++cmake_minimum_required ( VERSION 3.5 )
+ 
+ foreach (policy CMP0091 CMP0022)
+ 	if (POLICY ${policy})

pkgs/by-name/ma/manticoresearch/package.nix

@@ -1,40 +1,25 @@
 {
-  lib,
-  stdenv,
-  fetchFromGitHub,
   bison,
-  cmake,
-  flex,
-  pkg-config,
   boost,
+  callPackage,
+  cmake,
+  croaring,
+  fetchFromGitHub,
+  flex,
   icu,
+  lib,
   libstemmer,
-  mariadb-connector-c,
-  re2,
-  nlohmann_json,
-  testers,
   manticoresearch,
+  mariadb-connector-c,
+  nlohmann_json,
+  pkg-config,
+  re2,
+  stdenv,
+  testers,
 }:
 
 let
-  columnar = stdenv.mkDerivation (finalAttrs: {
-    pname = "columnar";
-    version = "c21-s10"; # see NEED_COLUMNAR_API/NEED_SECONDARY_API in Manticore's cmake/GetColumnar.cmake
-    src = fetchFromGitHub {
-      owner = "manticoresoftware";
-      repo = "columnar";
-      rev = finalAttrs.version;
-      hash = "sha256-TGFGFfoyHnPSr2U/9dpqFLUN3Dt2jDQrTF/xxDY4pdE=";
-    };
-    nativeBuildInputs = [ cmake ];
-    cmakeFlags = [ "-DAPI_ONLY=ON" ];
-    meta = {
-      description = "Column-oriented storage and secondary indexing library";
-      homepage = "https://github.com/manticoresoftware/columnar/";
-      license = lib.licenses.asl20;
-      platforms = lib.platforms.all;
-    };
-  });
+  columnar = callPackage ./columnar.nix { };
   uni-algo = stdenv.mkDerivation (finalAttrs: {
     pname = "uni-algo";
     version = "0.7.2";
@@ -52,16 +37,60 @@ let
       platforms = lib.platforms.all;
     };
   });
+  cctz = stdenv.mkDerivation {
+    pname = "manticore-cctz";
+    version = "0-unstable-2024-05-08";
+
+    src = fetchFromGitHub {
+      owner = "manticoresoftware";
+      repo = "cctz";
+      rev = "cf11f758e2532161c9e21c3ec2461b0fafb15853";
+      hash = "sha256-oXn6YowOg+9jaXXSX1fggkgE9o9xZ4hlmrpdpEHot68=";
+    };
+
+    patches = [ ./cctz-cmake-policy.patch ];
+
+    nativeBuildInputs = [ cmake ];
+    cmakeBuildDir = "build_dir"; # Avoid conflicts with the pre-existing `BUILD` file on case-insensitive FS
+
+    meta = {
+      description = "Library for translating between absolute and civil times using the rules of a time zone";
+      homepage = "https://github.com/manticoresoftware/cctz";
+      license = lib.licenses.asl20;
+      platforms = lib.platforms.all;
+    };
+  };
+  xxhash = stdenv.mkDerivation {
+    pname = "manticore-xxhash";
+    version = "0.8.2-unstable-2024-07-24";
+
+    src = fetchFromGitHub {
+      owner = "manticoresoftware";
+      repo = "xxHash";
+      rev = "72997b0070a031c063f86a308aec77ae742706d3";
+      hash = "sha256-QAIxZeMiohm/BYyO0f70En6GOv7t3yLH2pJfkUek7Js=";
+    };
+    nativeBuildInputs = [ cmake ];
+    meta = {
+      description = "Extremely fast non-cryptographic hash algorithm";
+      homepage = "https://github.com/manticoresoftware/xxhash";
+      license = with lib.licenses; [
+        bsd2
+        gpl2
+      ];
+      platforms = lib.platforms.all;
+    };
+  };
 in
 stdenv.mkDerivation (finalAttrs: {
   pname = "manticoresearch";
-  version = "6.2.12";
+  version = "15.1.0";
 
   src = fetchFromGitHub {
     owner = "manticoresoftware";
     repo = "manticoresearch";
     tag = finalAttrs.version;
-    hash = "sha256-UD/r7rlJ5mR3wg4doKT/nTwTWzlulngUjOPNEjmykB8=";
+    hash = "sha256-ESiM2D11o1QnctDzL7WQ+usad7nvs0YSPOpZzSfYv4Y=";
   };
 
   nativeBuildInputs = [
@@ -73,41 +102,54 @@ stdenv.mkDerivation (finalAttrs: {
 
   buildInputs = [
     boost
-    columnar
+    cctz
+    columnar.dev
+    croaring
     icu.dev
     libstemmer
     mariadb-connector-c
     nlohmann_json
     uni-algo
     re2
+    xxhash
   ];
 
   postPatch = ''
     sed -i 's/set ( Boost_USE_STATIC_LIBS ON )/set ( Boost_USE_STATIC_LIBS OFF )/' src/CMakeLists.txt
 
-    # supply our own packages rather than letting manticore download dependencies during build
-    sed -i 's/^with_get/with_menu/' CMakeLists.txt
-    sed -i 's/get_dep \( nlohmann_json .* \)/find_package(nlohmann_json)/' CMakeLists.txt
-    sed -i 's/get_dep \( uni-algo .* \)/find_package(uni-algo)/' CMakeLists.txt
+    # Skip jieba, it requires a bunch of additional dependencies
+    sed -i '/with_get ( jieba /d' CMakeLists.txt
+
+    # Fill in a version number for the VERNUMBERS macro
+    sed -i 's/0\.0\.0/${finalAttrs.version}/' src/sphinxversion.h.in
   '';
 
   cmakeFlags = [
     "-DWITH_GALERA=0"
+    "-DWITH_ICU=1"
+    # Supply our own packages rather than letting manticore download dependencies during build
+    "-DWITH_ICU_FORCE_STATIC=OFF"
+    "-DWITH_RE2_FORCE_STATIC=OFF"
+    "-DWITH_STEMMER_FORCE_STATIC=OFF"
     "-DWITH_MYSQL=1"
     "-DMYSQL_INCLUDE_DIR=${mariadb-connector-c.dev}/include/mariadb"
     "-DMYSQL_LIB=${mariadb-connector-c.out}/lib/mariadb/libmysqlclient.a"
     "-DCONFDIR=${placeholder "out"}/etc"
     "-DLOGDIR=/var/lib/manticoresearch/log"
     "-DRUNDIR=/var/run/manticoresearch"
+    "-DDISTR=nixpkgs"
   ];
 
   postFixup = ''
     mkdir -p $out/lib/systemd/system
     cp ${finalAttrs.src}/dist/deb/manticore.service.in $out/lib/systemd/system/manticore.service
     substituteInPlace $out/lib/systemd/system/manticore.service \
-      --replace "@CMAKE_INSTALL_FULL_RUNSTATEDIR@" "/var/lib/manticore" \
-      --replace "@CMAKE_INSTALL_FULL_BINDIR@" "$out/bin" \
-      --replace "@CMAKE_INSTALL_FULL_SYSCONFDIR@" "$out/etc"
+      --replace-fail "@CMAKE_INSTALL_FULL_RUNSTATEDIR@" "/var/lib/manticore" \
+      --replace-fail "@CMAKE_INSTALL_FULL_BINDIR@" "$out/bin" \
+      --replace-fail "@CMAKE_INSTALL_FULL_SYSCONFDIR@" "$out/etc"
+
+    mkdir $out/share/manticore/modules
+    cp ${columnar}/share/manticore/modules/* $out/share/manticore/modules
   '';
 
   passthru.tests.version = testers.testVersion {
@@ -120,7 +162,7 @@ stdenv.mkDerivation (finalAttrs: {
     description = "Easy to use open source fast database for search";
     homepage = "https://manticoresearch.com";
     changelog = "https://github.com/manticoresoftware/manticoresearch/releases/tag/${finalAttrs.version}";
-    license = lib.licenses.gpl2;
+    license = lib.licenses.gpl3Plus;
     mainProgram = "searchd";
     maintainers = [ lib.maintainers.jdelStrother ];
     platforms = lib.platforms.all;