diff --git a/flake.lock b/flake.lock index 6af940a3..d302469e 100644 --- a/flake.lock +++ b/flake.lock @@ -880,11 +880,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1751630609, - "narHash": "sha256-mJ1XnKiLnNapGSUwyGdFD8tmQTzuJm8z3qaFC27guqE=", + "lastModified": 1753842094, + "narHash": "sha256-1IAtQ0itZjT07XyUNfHJ4r8nO4+036T6scVUFs1NEko=", "owner": "CHN-beta", "repo": "nixpkgs", - "rev": "b263408b62d74a1e7a298fc47135653a70c227aa", + "rev": "2ebe9e65f6f1223450f58bf0197fcbce3fb9bf0f", "type": "github" }, "original": { diff --git a/flake/lib/buildNixpkgsConfig/boost188.patch b/flake/lib/buildNixpkgsConfig/boost188.patch new file mode 100644 index 00000000..5347879d --- /dev/null +++ b/flake/lib/buildNixpkgsConfig/boost188.patch @@ -0,0 +1,13 @@ +diff --git a/boost/process/v2/stdio.hpp b/boost/process/v2/stdio.hpp +index 01d0216..4084e46 100644 +--- a/boost/process/v2/stdio.hpp ++++ b/boost/process/v2/stdio.hpp +@@ -184,7 +184,7 @@ struct process_io_binding + process_io_binding & operator=(const process_io_binding &) = delete; + + process_io_binding(process_io_binding && other) noexcept +- : fd(other.fd), fd_needs_closing(other.fd), ec(other.ec) ++ : fd(other.fd), fd_needs_closing(other.fd_needs_closing), ec(other.ec) + { + other.fd = target; + other.fd_needs_closing = false; diff --git a/flake/lib/buildNixpkgsConfig/default.nix b/flake/lib/buildNixpkgsConfig/default.nix index 448bc82f..bef11f24 100644 --- a/flake/lib/buildNixpkgsConfig/default.nix +++ b/flake/lib/buildNixpkgsConfig/default.nix @@ -69,6 +69,7 @@ in platformConfig // patches = prev.patches or [] ++ [ ./root.patch ]; cmakeFlags = prev.cmakeFlags ++ [ "-DCMAKE_CXX_STANDARD=23" ]; }); + boost188 = prev.boost188.overrideAttrs (prev: { patches = prev.patches or [] ++ [ ./boost188.patch ]; }); inherit (final.pkgs-2411) iio-sensor-proxy; inherit (final.pkgs-unstable) bees; } diff --git a/packages/biu/CMakeLists.txt b/packages/biu/CMakeLists.txt index ce649d73..caf5108f 100644 --- a/packages/biu/CMakeLists.txt +++ b/packages/biu/CMakeLists.txt @@ -13,8 +13,8 @@ endif() find_package(magic_enum REQUIRED) find_package(fmt REQUIRED) -find_package(Boost REQUIRED COMPONENTS headers iostreams filesystem system process stacktrace_from_exception) -# stacktrace_backtrace +find_package(Boost REQUIRED COMPONENTS headers iostreams filesystem system process stacktrace_from_exception + stacktrace_backtrace) find_package(range-v3 REQUIRED) find_path(NAMEOF_INCLUDE_DIR nameof.hpp REQUIRED) find_package(Eigen3 REQUIRED) @@ -36,7 +36,7 @@ target_include_directories(biu PUBLIC $ # include # include +# include diff --git a/packages/biu/include/biu/atomic.hpp b/packages/biu/include/biu/atomic.hpp index af3d2aef..3c2da02c 100644 --- a/packages/biu/include/biu/atomic.hpp +++ b/packages/biu/include/biu/atomic.hpp @@ -37,7 +37,7 @@ namespace biu public: ValueType get(this auto&& self); public: operator ValueType(this auto&& self); - protected: template auto lock_(this auto&& self, auto&& condition_function, auto timeout); + protected: template auto lock_(this auto&& self, auto&& condition_function, auto timeout); // Apply a function to stored value. // Wait for some time (if provided) until condition funciton returns true (if provided) @@ -45,6 +45,7 @@ namespace biu // NoReturn: throw exception if timeout, ignore function result, and return *this, if true; // return bool or std::optional wrapped result of function, if false. // Useful when chaining multiple apply() calls. + // timeout 只考虑 condition_function 的超时，不考虑锁本身的超时。 public: template decltype(auto) apply (this auto&& self, auto&& function, auto&& condition_function, auto&& timeout); public: template decltype(auto) apply @@ -54,8 +55,7 @@ namespace biu // Wait until condition funciton returns true or *this, with an optional timeout public: template decltype(auto) wait (this auto&& self, auto&& condition_function, auto timeout); - public: template decltype(auto) wait - (this auto&& self, auto&& condition_function); + public: decltype(auto) wait(this auto&& self, auto&& condition_function); // Attain lock from outside when constructing, and release when destructing. // Throw: same effect as NoReturn. diff --git a/packages/biu/include/biu/atomic.tpp b/packages/biu/include/biu/atomic.tpp index 0ba46094..21f6637f 100644 --- a/packages/biu/include/biu/atomic.tpp +++ b/packages/biu/include/biu/atomic.tpp @@ -41,8 +41,11 @@ namespace biu (this auto&& self, auto&& condition_function, auto timeout) { if constexpr (Nullptr) + { + static_assert(Nullptr); return Guard> (std::unique_lock{self.Mutex_}, std::experimental::make_observer(&self), {}); + } else if constexpr (Nullptr) { std::unique_lock lock(self.Mutex_); @@ -70,28 +73,35 @@ namespace biu using function_return_type = std::invoke_result_t>; auto&& lock = std::forward(self).template lock_ (std::forward(condition_function), timeout); - // 如果得到的是 optional + // 如果 lock 是 optional if constexpr (SpecializationOf, std::optional>) - // 如果超时了，返回 false 或者对应的 nullopt + // 如果超时了，这时 NoReturn 一定是 false，返回 false 或者对应的 nullopt if (!lock) + { + static_assert(!NoReturn); if constexpr (std::is_void_v) return false; else return std::optional(); + } // 否则，执行函数 else + // 如果函数本身返回 void，则返回 true 或者 *this if constexpr (std::is_void_v) { std::forward(function) (std::forward>(self.Value_)); - // 如果函数本身返回 void 并且不可能超时，返回 *this，否则返回 true - if constexpr (Nullptr || Nullptr) - return std::forward(self); + // 如果要求不返回结果，则返回 *this + if constexpr (NoReturn) return std::forward(self); + // 否则，返回 true else return true; } + // 否则，返回函数的返回值或者 std::optional 包装的结果 else { auto&& result = std::forward(function) (std::forward>(self.Value_)); - return std::make_optional(std::forward(result)); + if constexpr (NoReturn) + return std::forward(self); + else return std::make_optional(std::forward(result)); } // 否则，说明不可能超时，返回函数的返回值或者 *this else @@ -125,15 +135,15 @@ namespace biu template template decltype(auto) Atomic::wait (this auto&& self, auto&& condition_function, auto timeout) { - auto result = std::forward(self).template lock_ + auto&& result = std::forward(self).template lock_ (std::forward(condition_function), timeout); if constexpr (SpecializationOf) return result.has_value(); - else return std::forward(result); + else return std::forward(self); } - template template decltype(auto) Atomic::wait + template decltype(auto) Atomic::wait (this auto&& self, auto&& condition_function) { - return std::forward(self).template wait + return std::forward(self).template wait (std::forward(condition_function), nullptr); } diff --git a/packages/biu/include/biu/common.hpp b/packages/biu/include/biu/common.hpp index 1ea2a987..7ae93090 100644 --- a/packages/biu/include/biu/common.hpp +++ b/packages/biu/include/biu/common.hpp @@ -74,27 +74,6 @@ namespace biu template using FallbackIfNoTypeDeclared = typename detail_::FallbackIfNoTypeDeclaredHelper::Type; - namespace detail_ - { - struct ExecMode { bool DirectStdin = false, DirectStdout = false, DirectStderr = false, SearchPath = false; }; - template struct ExecResult - { - int ExitCode; - std::conditional_t Stdout; - std::conditional_t Stderr; - operator bool() const; - }; - template struct ExecInput - { - std::conditional_t Program; - std::vector Args; - std::conditional_t Stdin = {}; - std::map ExtraEnv = {}; - std::optional Timeout; - }; - } - template detail_::ExecResult exec(detail_::ExecInput input); - template concurrencpp::generator> sequence(Array from, Array to); template concurrencpp::generator> sequence(Array to); @@ -120,9 +99,11 @@ namespace biu constexpr detail_::ToLvalueHelper toLvalue; template void for_each(Function&& function, T&& arg, Ts&&... args); + + template decltype(auto) perfect_return(T&& obj); } using common::hash, common::unused, common::block_forever, common::is_interactive, common::env, common::int128_t, common::uint128_t, common::Empty, common::CaseInsensitiveStringLessComparator, common::RemoveMemberPointer, - common::MoveQualifiers, common::FallbackIfNoTypeDeclared, common::exec, common::sequence, common::read, - common::toLvalue, common::for_each; + common::MoveQualifiers, common::FallbackIfNoTypeDeclared, common::sequence, common::read, + common::toLvalue, common::for_each, common::perfect_return; } diff --git a/packages/biu/include/biu/common.tpp b/packages/biu/include/biu/common.tpp index ea862e17..5d91cd0e 100644 --- a/packages/biu/include/biu/common.tpp +++ b/packages/biu/include/biu/common.tpp @@ -75,4 +75,18 @@ namespace biu::common ); } } + + template decltype(auto) perfect_return(T&& obj) + { + // 不允许返回右值引用 + static_assert(!std::is_rvalue_reference_v); + // 左值引用则返回左值引用 + if constexpr (std::is_lvalue_reference_v) return (obj); + // 否则，假定可以移动，并返回值 + else + { + static_assert(std::is_move_constructible_v>); + return std::move(obj); + } + } } diff --git a/packages/biu/include/biu/process.hpp b/packages/biu/include/biu/process.hpp new file mode 100644 index 00000000..7ed08954 --- /dev/null +++ b/packages/biu/include/biu/process.hpp @@ -0,0 +1,36 @@ +# pragma once +# include + +namespace biu +{ + namespace process + { + enum class IoType { Direct, Close, String }; + namespace detail_ + { + struct ExecMode + { + bool SearchPath = false, ModifyEnv = false, Timeout = false; + IoType Stdin = IoType::Direct, Stdout = IoType::Direct, Stderr = IoType::Direct; + }; + template struct ExecResult + { + int ExitCode; + std::conditional_t Stdout; + std::conditional_t Stderr; + operator bool() const; + }; + template struct ExecInput + { + std::conditional_t Program; + std::vector Args; + std::conditional_t Stdin = {}; + std::conditional_t, Empty> ExtraEnv = {}; + std::conditional_t Timeout = {}; + }; + } + template + detail_::ExecResult exec(detail_::ExecInput input, Ts&&... args); + } + using process::exec, process::IoType; +} diff --git a/packages/biu/include/biu/process.tpp b/packages/biu/include/biu/process.tpp new file mode 100644 index 00000000..b9e3cb21 --- /dev/null +++ b/packages/biu/include/biu/process.tpp @@ -0,0 +1,178 @@ +# pragma once +# include +# include +# include +# include +# include + +namespace biu::process +{ + template detail_::ExecResult::operator bool() const { return ExitCode == 0; } + + template detail_::ExecResult exec + (detail_::ExecInput input, Ts&&... args) + { + Logger::Guard log; + namespace bp = boost::process; + boost::asio::io_context context; + detail_::ExecResult result; + using namespace biu::literals; + + // 进程是在创建时就开始运行的，而不是在 io_context.run() 时才开始运行 + + // seach actual program + boost::filesystem::path actual_program = [&] + { + if constexpr (Mode.SearchPath) return bp::environment::find_executable(input.Program); + else return input.Program.string(); + }(); + log.debug("Searching for program: {} -> {}"_f(input.Program, actual_program.string())); + + // env + auto env = [&] + { + if constexpr (Mode.ModifyEnv) + { + auto current = bp::environment::current(); + std::unordered_map env + (current.begin(), current.end()); + for (const auto& [key, value] : input.ExtraEnv) env[key] = value; + } + else return bp::environment::current(); + }(); + log(); + + // prepare io pipes + std::unique_ptr stdin_pipe; + std::unique_ptr stdout_pipe, stderr_pipe; + std::function read_some = + [&read_some](auto& p, auto& result) + { + auto buffer = std::make_shared>(); + p.async_read_some + ( + boost::asio::buffer(*buffer), + [&, buffer](const boost::system::error_code& ec, std::size_t len) + { + Logger::Guard log; + if (!ec) { result.append(buffer->data(), len); read_some(p, result); log.debug("read {}"_f(len)); } + else log.debug("Error reading from pipe: {} {}"_f(ec.value(), ec.message())); + } + ); + }; + std::function write_some = + [&write_some](auto& p, auto& result) + { + if (result.empty()) { p.close(); return; } + auto buffer = std::make_shared>(); + std::copy(result.begin(), result.end(), buffer->begin()); + p.async_write_some + ( + boost::asio::buffer(*buffer), + [&, buffer](const auto& ec, std::size_t len) + { + Logger::Guard log; + if (!ec) { result.erase(0, len); write_some(p, result); log.debug("write {}"_f(len)); } + else log.debug("Error reading from pipe: {} {}"_f(ec.value(), ec.message())); + } + ); + }; + bp::process_stdio stdio + { + .in = [&] -> decltype(auto) + { + if constexpr (Mode.Stdin == IoType::Close) return nullptr; + else if constexpr (Mode.Stdin == IoType::Direct) + return decltype(bp::process_stdio::in)(); + else if constexpr (Mode.Stdin == IoType::String) + { + stdin_pipe = std::make_unique(context); + write_some(*stdin_pipe, input.Stdin); + return (*stdin_pipe); + } + else std::unreachable(); + }(), + .out = [&] -> decltype(auto) + { + if constexpr (Mode.Stdout == IoType::Close) return nullptr; + else if constexpr (Mode.Stdout == IoType::Direct) + return decltype(bp::process_stdio::out)(); + else if constexpr (Mode.Stdout == IoType::String) + { + stdout_pipe = std::make_unique(context); + read_some(*stdout_pipe, result.Stdout); + return (*stdout_pipe); + } + else std::unreachable(); + }(), + .err = [&] -> decltype(auto) + { + if constexpr (Mode.Stderr == IoType::Close) return nullptr; + else if constexpr (Mode.Stderr == IoType::Direct) + return decltype(bp::process_stdio::err)(); + else if constexpr (Mode.Stderr == IoType::String) + { + stderr_pipe = std::make_unique(context); + read_some(*stderr_pipe, result.Stderr); + return (*stderr_pipe); + } + else std::unreachable(); + }() + }; + log(); + + // start process + if constexpr (Mode.Timeout) + { + boost::asio::steady_timer timeout{context, input.Timeout}; + boost::asio::cancellation_signal sig; + Atomic finished{false}; + Logger::try_exec([&] + { + auto proc = bp::process + (context, actual_program, input.Args, std::move(stdio), std::move(env), std::forward(args)...); + bp::async_execute + ( + std::move(proc), + boost::asio::bind_cancellation_slot + ( + sig.slot(), + [&](bp::v2::error_code ec, int exit_code) + { + result.ExitCode = exit_code; + timeout.cancel(); + finished = true; + } + ) + ); + timeout.expires_after(input.Timeout); + timeout.async_wait([&](auto ec) + { + if (ec) return; + sig.emit(boost::asio::cancellation_type::partial); + timeout.expires_after(input.Timeout); + timeout.async_wait + ([&](auto ec) { if (!ec) sig.emit(boost::asio::cancellation_type::terminal); }); + }); + context.run(); + finished.wait([](auto& v) { return v; }); + }); + } + else + { + auto thread = std::thread([&] + { + Logger::try_exec([&] + { + auto proc = bp::process + (context, actual_program, input.Args, std::move(stdio), std::move(env), std::forward(args)...); + proc.wait(); + result.ExitCode = proc.exit_code(); + }); + }); + context.run(); + thread.join(); + } + return result; + } +} diff --git a/packages/biu/src/common.cpp b/packages/biu/src/common.cpp index efc92186..587973fb 100644 --- a/packages/biu/src/common.cpp +++ b/packages/biu/src/common.cpp @@ -3,8 +3,6 @@ # include # define BIU_INTERNAL # include -# include -# include namespace biu { @@ -19,56 +17,6 @@ namespace biu else return value; } - template detail_::ExecResult::operator bool() const { return ExitCode == 0; } - - template detail_::ExecResult exec(detail_::ExecInput input) - { - // TODO: switch to v2 - namespace bp = boost::process; - - // decide input/output format, prepare environment, seach actual program - bp::ipstream stdout_stream, stderr_stream; - bp::opstream input_stream; - auto&& stdin_format = [&] - { if constexpr (Mode.DirectStdin) return bp::std_in < stdin; else return bp::std_in < input_stream; }(); - auto&& stdout_format = [&] - { if constexpr (Mode.DirectStdout) return bp::std_out > stdout; else return bp::std_out > stdout_stream; }(); - auto&& stderr_format = [&] - { if constexpr (Mode.DirectStderr) return bp::std_err > stderr; else return bp::std_err > stderr_stream; }(); - auto&& actual_program = [&] - { - if constexpr (Mode.SearchPath) return bp::search_path(input.Program); - else return input.Program.string(); - }(); - bp::environment env = boost::this_process::environment(); - for (const auto& [key, value] : input.ExtraEnv) env[key] = value; - - // start - auto process = bp::child - (actual_program, bp::args(input.Args), stdout_format, stderr_format, stdin_format, env); - if constexpr (!Mode.DirectStdin) { input_stream << input.Stdin; input_stream.pipe().close(); } - - // wait for exit - if (input.Timeout) { if (!process.wait_for(*input.Timeout)) process.terminate(); } - else process.wait(); - - // collect output - detail_::ExecResult result; - result.ExitCode = process.exit_code(); - if constexpr (!Mode.DirectStdout) result.Stdout = {std::istreambuf_iterator{stdout_stream.rdbuf()}, {}}; - if constexpr (!Mode.DirectStderr) result.Stderr = {std::istreambuf_iterator{stderr_stream.rdbuf()}, {}}; - return result; - } - -# define BIU_EXEC_PRED(r, i) BOOST_PP_NOT_EQUAL(i, 16) -# define BIU_EXEC_OP(r, i) BOOST_PP_INC(i) -# define BIU_EXEC_MACRO(r, i) \ - namespace detail_ \ - { constexpr ExecMode ExecMode##i {(i & 1) != 0, (i & 2) != 0, (i & 4) != 0, (i & 8) != 0}; } \ - template detail_::ExecResult::operator bool() const; \ - template detail_::ExecResult \ - exec(detail_::ExecInput); - BOOST_PP_FOR(0, BIU_EXEC_PRED, BIU_EXEC_OP, BIU_EXEC_MACRO) template<> std::vector read(const std::filesystem::path& path) { auto length = std::filesystem::file_size(path); diff --git a/packages/biu/test/process.cpp b/packages/biu/test/process.cpp index 6540c301..36f6a543 100644 --- a/packages/biu/test/process.cpp +++ b/packages/biu/test/process.cpp @@ -3,7 +3,8 @@ int main() { using namespace biu::literals; - auto result = biu::exec<{.SearchPath = true}>({.Program = "sleep", .Args = {"10"}, .Timeout = 3s}); + biu::Logger::Guard log; + auto result = biu::exec<{.SearchPath = true, .Timeout = true}>({.Program = "sleep", .Args = {"10"}, .Timeout = 3s}); std::cout << "{}\n"_f(result.ExitCode); assert(!result); } diff --git a/packages/default.nix b/packages/default.nix index 1de87f9e..0a35680f 100644 --- a/packages/default.nix +++ b/packages/default.nix @@ -76,9 +76,8 @@ inputs: rec # TODO: report glaze bug to upstream inherit (inputs.pkgs.pkgs-2411) glaze; stdenv = inputs.pkgs.clang18Stdenv; - boost = inputs.pkgs.boost187; - # boost = (inputs.pkgs.boost188.override { extraB2Args = [ "boost.stacktrace.backtrace=on" ]; }).overrideAttrs - # (prev: { buildInputs = prev.buildInputs ++ [(inputs.pkgs.libbacktrace.override { enableStatic = true; })]; }); + boost = (inputs.pkgs.boost188.override { extraB2Args = [ "boost.stacktrace.backtrace=on" ]; }).overrideAttrs + (prev: { buildInputs = prev.buildInputs ++ [(inputs.pkgs.libbacktrace.override { enableStatic = true; })]; }); fmt = inputs.pkgs.fmt_11.overrideAttrs (prev: { patches = prev.patches or [] ++ [ ./biu/fmt.patch ]; }); }; hpcstat = inputs.pkgs.callPackage ./hpcstat diff --git a/packages/hpcstat/src/disk.cpp b/packages/hpcstat/src/disk.cpp index c8ba42bc..cc517318 100644 --- a/packages/hpcstat/src/disk.cpp +++ b/packages/hpcstat/src/disk.cpp @@ -27,7 +27,7 @@ namespace hpcstat::disk { std::cerr << "HPCSTAT_DATADIR not set\n"; return false; } else if ( - auto result = biu::exec<{.DirectStdout = true, .DirectStderr = true}> + auto result = biu::exec ({ // duc index -d ./duc.db -p ~ "{}/duc"_f(*ducbindir), diff --git a/packages/hpcstat/src/lfs.cpp b/packages/hpcstat/src/lfs.cpp index 54468be7..ea444d2d 100644 --- a/packages/hpcstat/src/lfs.cpp +++ b/packages/hpcstat/src/lfs.cpp @@ -44,7 +44,7 @@ namespace hpcstat::lfs { if ( - auto result = biu::exec<{.SearchPath = true}> + auto result = biu::exec<{.SearchPath = true, .ModifyEnv = true}> ({ .Program="bjobs", .Args={ "-a", "-o", "jobid submit_time stat cpu_used job_name", "-json" }, diff --git a/packages/hpcstat/src/ssh.cpp b/packages/hpcstat/src/ssh.cpp index 0061c4e7..8fe63cbb 100644 --- a/packages/hpcstat/src/ssh.cpp +++ b/packages/hpcstat/src/ssh.cpp @@ -12,7 +12,7 @@ namespace hpcstat::ssh return std::nullopt; else if ( - auto output = biu::exec + auto output = biu::exec<{.Timeout = true}> ({.Program=std::filesystem::path(*sshbindir) / "ssh-add", .Args{ "-l" }, .Timeout=10s}); !output ) @@ -41,7 +41,7 @@ namespace hpcstat::ssh return std::nullopt; else if ( - auto output = biu::exec + auto output = biu::exec<.Stdin = biu::IoType::String, .Timeout = true> ({ .Program=std::filesystem::path(*sshbindir) / "ssh-keygen", .Args={ @@ -69,7 +69,7 @@ namespace hpcstat::ssh bf::create_directories(tempdir); auto signaturefile = tempdir / "signature"; std::ofstream(signaturefile) << signature; - auto result = biu::exec + auto result = biu::exec<.Stdin = biu::IoType::String, .Timeout = true> ({ .Program=std::filesystem::path(*sshbindir) / "ssh-keygen", .Args={ diff --git a/packages/sbatch-tui/src/main.cpp b/packages/sbatch-tui/src/main.cpp index 1e360b0e..3f6e290a 100644 --- a/packages/sbatch-tui/src/main.cpp +++ b/packages/sbatch-tui/src/main.cpp @@ -149,7 +149,7 @@ int main() catch (...) {} // 提交任务 boost::replace_all(State.SubmitCommand, "\n", " "); - biu::exec<{.DirectStdout = true, .DirectStderr = true, .SearchPath = true}> + biu::exec<{.SearchPath = true}> ({"sh", { "-c", State.SubmitCommand }}); break; } diff --git a/packages/vm/src/main.cpp b/packages/vm/src/main.cpp index a614788e..51685a4b 100644 --- a/packages/vm/src/main.cpp +++ b/packages/vm/src/main.cpp @@ -55,8 +55,7 @@ vm force-reboot if (!config.vm.contains(uid)) throw std::runtime_error("No VM found for current user"); if (!config.vm[uid].contains(args[1])) throw std::runtime_error("VM {} is not owned by current user"_f(args[1])); - biu::exec<{.DirectStdout = true, .DirectStderr = true, .SearchPath = false}> - ({config.virsh, { vm_to_virsh[args[0]], args[1] }}); + biu::exec({config.virsh, { vm_to_virsh[args[0]], args[1] }}); } else throw std::runtime_error("unknown command: {}"_f(args[0])); });