chn/nixos
1
0
Fork 0
mirror of https://github.com/CHN-beta/nixos.git synced 2026-01-12 04:19:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

localPackages.hpcstat: use biu to format

Browse Source
This commit is contained in:
陈浩南
2024-06-10 20:18:56 +08:00
parent 076a267ba4
commit 423debe893
11 changed files with 64 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
local/pkgs/biu/include/biu/format.hpp
View File

@@ -50,8 +50,6 @@ namespace biu
namespace fmt
{
using namespace biu::stream_operators;
template <typename Char, biu::detail_::OptionalWrap Wrap> struct formatter<Wrap, Char>
: biu::detail_::FormatterReuseProxy<typename biu::detail_::UnderlyingTypeOfOptionalWrap<Wrap>::Type>
{

65
local/pkgs/biu/include/biu/format.tpp
View File

@@ -1,15 +1,16 @@
# pragma once
# include <nameof.hpp>
# include <biu/format.hpp>
# include <fmt/core.h>
namespace biu
{
template <typename Char, Char... c> template <typename... Param>
std::basic_string<Char> detail_::FormatLiteralHelper<Char, c...>::operator() (Param&&... param) const
{return fmt::format(BasicStaticString<Char, c...>::StringView, std::forward<Param>(param)...);}
{ return fmt::format(BasicStaticString<Char, c...>::StringView, std::forward<Param>(param)...); }
template <typename Char, Char... c> consteval
detail_::FormatLiteralHelper<Char, c...> literals::operator""_f()
{return {};}
{ return {}; }
template <typename T> constexpr
auto detail_::FormatterReuseProxy<T>::parse(fmt::format_parse_context& ctx)
@@ -32,10 +33,8 @@ namespace biu
{
if (holds_alternative<T>(value))
{
if constexpr (biu::Formattable<T, Char>)
os << "({}: {})"_f(nameof::nameof_full_type<T>(), get<T>(value));
else
os << "({}: {})"_f(nameof::nameof_full_type<T>(), "non-null unformattable value");
if constexpr (biu::Formattable<T, Char>) os << "({}: {})"_f(nameof::nameof_full_type<T>(), get<T>(value));
else os << "({}: {})"_f(nameof::nameof_full_type<T>(), "non-null unformattable value");
}
};
(try_print.template operator()<Ts>(), ...);
@@ -49,61 +48,37 @@ namespace fmt
auto formatter<Wrap, Char>::format(const Wrap& wrap, FormatContext& ctx)
-> std::invoke_result_t<decltype(&FormatContext::out), FormatContext>
{
using namespace biu::literals;
using namespace biu::stream_operators;
using value_t = biu::detail_::UnderlyingTypeOfOptionalWrap<Wrap>::Type;
auto format_value_type = [&, this](const value_t& value)
{
if constexpr (!biu::Formattable<value_t, Char>)
return format_to(ctx.out(), "non-null unformattable value");
if constexpr (!biu::Formattable<value_t, Char>) return fmt::format_to(ctx.out(), "non-null unformattable value");
else if constexpr (std::default_initializable<formatter<value_t>>)
biu::detail_::FormatterReuseProxy<value_t>::format(value, ctx);
else
format_to(ctx.out(), "{}", value);
else fmt::format_to(ctx.out(), "{}", value);
};
format_to(ctx.out(), "(");
fmt::format_to(ctx.out(), "(");
if constexpr (biu::SpecializationOf<Wrap, std::optional>)
{
if (wrap)
format_value_type(*wrap);
else
format_to(ctx.out(), "null");
}
{ if (wrap) format_value_type(*wrap); else fmt::format_to(ctx.out(), "null"); }
else if constexpr (biu::SpecializationOf<Wrap, std::weak_ptr>)
{
if (auto shared = wrap.lock())
{
format_to(ctx.out(), "{} ", ptr(shared.get()));
format_value_type(*shared);
}
else
format_to(ctx.out(), "null");
{ fmt::format_to(ctx.out(), "{} ", ptr(shared.get())); format_value_type(*shared); }
else fmt::format_to(ctx.out(), "null");
}
else
{
if (wrap)
{
format_to(ctx.out(), "{} ", ptr(wrap.get()));
format_value_type(*wrap);
}
else
format_to(ctx.out(), "null");
if (wrap) { fmt::format_to(ctx.out(), "{} ", ptr(wrap.get())); format_value_type(*wrap); }
else fmt::format_to(ctx.out(), "null");
}
return format_to(ctx.out(), ")");
return fmt::format_to(ctx.out(), ")");
}
template <typename Char, biu::Enumerable T> constexpr
auto formatter<T, Char>::parse(format_parse_context& ctx)
template <typename Char, biu::Enumerable T> constexpr auto formatter<T, Char>::parse(format_parse_context& ctx)
-> std::invoke_result_t<decltype(&format_parse_context::begin), format_parse_context>
{
auto it = ctx.begin();
if (it != ctx.end() && *it == 'f')
{
full = true;
it++;
}
if (it != ctx.end() && *it != '}')
throw format_error{"syntax error."};
if (it != ctx.end() && *it == 'f') { full = true; it++; }
if (it != ctx.end() && *it != '}') throw format_error{"syntax error."};
return it;
}
@@ -111,9 +86,7 @@ namespace fmt
auto formatter<T, Char>::format(const T& value, FormatContext& ctx)
-> std::invoke_result_t<decltype(&FormatContext::out), FormatContext>
{
if (full)
return format_to(ctx.out(), "{}::{}", nameof::nameof_type<T>(), nameof::nameof_enum(value));
else
return format_to(ctx.out(), "{}", nameof::nameof_enum(value));
if (full) return fmt::format_to(ctx.out(), "{}::{}", nameof::nameof_type<T>(), nameof::nameof_enum(value));
else return fmt::format_to(ctx.out(), "{}", nameof::nameof_enum(value));
}
}

3
local/pkgs/hpcstat/CMakeLists.txt
View File

@@ -11,7 +11,6 @@ endif()
set(HPCSTAT_VERSION "unknown" CACHE STRING "Version of the hpcstat")
find_package(fmt REQUIRED)
find_package(Boost REQUIRED COMPONENTS headers filesystem)
find_package(SqliteOrm REQUIRED)
find_package(nlohmann_json REQUIRED)
@@ -28,7 +27,7 @@ add_executable(hpcstat src/main.cpp src/env.cpp src/keys.cpp src/ssh.cpp src/sql
src/push.cpp src/disk.cpp)
target_compile_features(hpcstat PRIVATE cxx_std_23)
target_include_directories(hpcstat PRIVATE ${PROJECT_SOURCE_DIR}/include ${ZPP_BITS_INCLUDE_DIR})
target_link_libraries(hpcstat PRIVATE fmt::fmt Boost::headers Boost::filesystem sqlite_orm::sqlite_orm
target_link_libraries(hpcstat PRIVATE Boost::headers Boost::filesystem sqlite_orm::sqlite_orm
nlohmann_json::nlohmann_json range-v3::range-v3 date::date date::date-tz OpenXLSX::OpenXLSX httplib::httplib
termcolor::termcolor biu::biu)
target_compile_definitions(hpcstat PRIVATE HPCSTAT_VERSION="${HPCSTAT_VERSION}")

1
local/pkgs/hpcstat/include/hpcstat/common.hpp
View File

@@ -12,7 +12,6 @@
# include <chrono>
# include <fstream>
# include <future>
# include <fmt/format.h>
# include <date/date.h>
# include <date/tz.h>
# include <boost/interprocess/sync/file_lock.hpp>

10
local/pkgs/hpcstat/src/disk.cpp
View File

@@ -67,12 +67,12 @@ namespace hpcstat::disk
);
!result
)
{ std::cerr << fmt::format("failed to ls {}\n", path.value_or("home")); return {}; }
{ std::cerr << "failed to ls {}\n"_f(path); return {}; }
else
{
std::smatch match;
if (!std::regex_search(result.Stdout, match, std::regex(R"((\d+))")))
{ std::cerr << fmt::format("failed to parse {}\n", result.Stdout); return std::nullopt; }
{ std::cerr << "failed to parse {}\n"_f(result.Stdout); return std::nullopt; }
return std::stod(match[1]) / 1024 / 1024 / 1024;
}
};
@@ -93,13 +93,13 @@ namespace hpcstat::disk
("{}/duc"_f(*ducbindir), { "info", "-d", "{}/duc.db"_f(*datadir) });
!result
)
{ std::cerr << fmt::format("failed to get duc info\n"); return {}; }
{ std::cerr << "failed to get duc info\n"; return {}; }
else
{
std::smatch match;
// search string like 2024-06-08 13:45:19
if (!std::regex_search(result.Stdout, match, std::regex(R"((\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}))")))
{ std::cerr << fmt::format("failed to parse {}\n", result.Stdout); return {}; }
{ std::cerr << "failed to parse {}\n"_f(result.Stdout); return {}; }
return match[1];
}
};
@@ -111,7 +111,7 @@ namespace hpcstat::disk
for (const auto& [dir, recursive] : Directories)
{
if (!std::filesystem::exists(*homedir + "/" + dir))
{ std::cerr << fmt::format("{} does not exist\n", *homedir + "/" + dir); continue; }
{ std::cerr << "{} does not exist\n"_f(*homedir + "/" + dir); continue; }
if (auto size = get_size(dir)) usage.Teacher.push_back({ dir, *size });
else return {};
if (recursive) for (const auto& subdir : get_subdir(dir))

2
local/pkgs/hpcstat/src/env.cpp
View File

@@ -8,7 +8,7 @@ namespace hpcstat::env
{
if (auto value = std::getenv(name.c_str()); !value)
{
if (required) std::cerr << fmt::format("Failed to get environment variable {}\n", name);
if (required) std::cerr << "Failed to get environment variable {}\n"_f(name);
return std::nullopt;
}
else return value;

22
local/pkgs/hpcstat/src/lfs.cpp
View File

@@ -17,7 +17,7 @@ namespace hpcstat::lfs
{
if (!valid_args.contains(it->substr(1)))
{
std::cerr << fmt::format("Unknown bsub argument: {}\n", *it)
std::cerr << "Unknown bsub argument: {}\n"_f(*it)
<< "bsub might support this argument, but hpcstat currently does not support it.\n"
"If you are sure this argument is supported by bsub,\n"
"please submit issue on [github](https://github.com/CHN-beta/hpcstat) or contact chn@chn.moe.\n";
@@ -35,11 +35,7 @@ namespace hpcstat::lfs
std::smatch match;
if (std::regex_search(result.Stdout, match, re))
return std::make_pair(std::stoi(match[1]), match[2]);
else
{
std::cerr << fmt::format("Failed to parse job id from output: {}\n", result.Stdout);
return std::nullopt;
}
else { std::cerr << "Failed to parse job id from output: {}\n"_f(result.Stdout); return std::nullopt; }
}
}
}
@@ -61,10 +57,7 @@ namespace hpcstat::lfs
nlohmann::json j;
try { j = nlohmann::json::parse(result.Stdout); }
catch (nlohmann::json::parse_error& e)
{
std::cerr << fmt::format("Failed to parse bjobs output: {}\n", e.what());
return std::nullopt;
}
{ std::cerr << "Failed to parse bjobs output: {}\n"_f(e.what()); return std::nullopt; }
std::map<unsigned, std::tuple<std::string, std::string, double, std::string>> jobs;
for (auto& job : j["RECORDS"])
{
@@ -76,7 +69,7 @@ namespace hpcstat::lfs
{
try { cpu_used = std::stof(cpu_used_str.substr(0, cpu_used_str.find(' '))); }
catch (std::invalid_argument& e)
{ std::cerr << fmt::format("Failed to parse cpu used: {}\n", e.what()); return std::nullopt; }
{ std::cerr << "Failed to parse cpu used: {}\n"_f(e.what()); return std::nullopt; }
}
jobs[std::stoi(job["JOBID"].get<std::string>())] =
{ job["SUBMIT_TIME"], status, cpu_used, job["JOB_NAME"] };
@@ -86,12 +79,7 @@ namespace hpcstat::lfs
}
std::optional<std::string> bjobs_detail(unsigned jobid)
{
if
(
auto result = biu::exec<{.SearchPath = true}>
("bjobs", { "-l", std::to_string(jobid) });
!result
)
if (auto result = biu::exec<{.SearchPath = true}>("bjobs", { "-l", "{}"_f(jobid) }); !result)
return std::nullopt;
else return result.Stdout;
}

28
local/pkgs/hpcstat/src/main.cpp
View File

@@ -54,11 +54,8 @@ int main(int argc, const char** argv)
sql::writedb(data);
if (env::interactive())
{
std::cout << fmt::format
(
"\33[2K\rLogged in as {} (Fingerprint: SHA256:{}{}).\n", Keys[*fp].Username, *fp,
sub_account ? fmt::format(" Subaccount {}", *sub_account) : ""
);
std::cout << "\33[2K\rLogged in as {} (Fingerprint: SHA256:{}{}).\n"_f
(Keys[*fp].Username, *fp, sub_account ? " Subaccount {}"_f(*sub_account) : "");
if (auto disk_stat = disk::get(); !disk_stat)
std::cerr << "Failed to get disk usage statistic.\n";
else
@@ -71,16 +68,16 @@ int main(int argc, const char** argv)
std::cout
<< color << "disk usage: " << termcolor::reset
<< bgcolor << termcolor::white
<< fmt::format("{:.1f}% ({:.1f}GB / ~800GB)", percent, disk_stat->Total) << termcolor::reset
<< color << fmt::format(" (estimated, counted at {})\n", disk_stat->Time) << termcolor::reset;
<< "{:.1f}% ({:.1f}GB / ~800GB)"_f(percent, disk_stat->Total) << termcolor::reset
<< color << " (estimated, counted at {})\n"_f(disk_stat->Time) << termcolor::reset;
if (percent > 80)
{
std::cout << color << "Top 3 directories owned by teacher:\n";
for (auto& [name, size] : disk_stat->Teacher | ranges::views::take(3))
std::cout << fmt::format(" {:.1f}GB {}\n", size, name);
std::cout << " {:.1f}GB {}\n"_f(size, name);
std::cout << color << "Top 3 directories owned by student:\n";
for (auto& [name, size] : disk_stat->Student | ranges::views::take(3))
std::cout << fmt::format(" {:.1f}GB {}\n", size, name);
std::cout << " {:.1f}GB {}\n"_f(size, name);
std::cout << termcolor::reset;
}
}
@@ -116,8 +113,7 @@ int main(int argc, const char** argv)
data.Signature = *signature;
lock.lock();
sql::writedb(data);
std::cout << fmt::format
("Job <{}> was submitted to <{}> by <{}>.\n", bsub->first, bsub->second, Keys[*fp].Username);
std::cout << "Job <{}> was submitted to <{}> by <{}>.\n"_f(bsub->first, bsub->second, Keys[*fp].Username);
}
}
else if (args[1] == "finishjob")
@@ -165,7 +161,7 @@ int main(int argc, const char** argv)
if (auto db_verify_result = sql::verify(args[2], args[3]); !db_verify_result) return 1;
else for (auto& data : *db_verify_result)
if (!std::apply(ssh::verify, data))
{ std::cerr << fmt::format("Failed to verify data: {}\n", std::get<0>(data)); return 1; }
{ std::cerr << "Failed to verify data: {}\n"_f(std::get<0>(data)); return 1; }
}
else if (args[1] == "export")
{
@@ -176,11 +172,7 @@ int main(int argc, const char** argv)
auto end = sys_seconds(sys_days(month(month_n) / 1 / year_n + months(1)))
.time_since_epoch().count();
lock.lock();
if
(
!sql::export_data
(begin, end, fmt::format("hpcstat-{}-{}.xlsx", year_n, month_n))
)
if (!sql::export_data(begin, end, "hpcstat-{}-{}.xlsx"_f(year_n, month_n)))
return 1;
}
else if (args[1] == "push")
@@ -195,7 +187,7 @@ int main(int argc, const char** argv)
auto stat_thread = std::async(std::launch::async, []{ return disk::stat(); });
std::cout << "Waiting for disk usage statistic to be collected... 0s" << std::flush;
for (unsigned i = 1; stat_thread.wait_for(1s) != std::future_status::ready; i++)
std::cout << fmt::format("\rWaiting for disk usage statistic to be collected... {}s", i) << std::flush;
std::cout << "\rWaiting for disk usage statistic to be collected... {}s"_f(i) << std::flush;
if (!stat_thread.get()) { std::cerr << "Failed to collect disk usage statistic.\n"; return 1; }
}
else { std::cerr << "Unknown command.\n"; return 1; }

20
local/pkgs/hpcstat/src/push.cpp
View File

@@ -14,7 +14,7 @@ namespace hpcstat::push
// 读取配置
if (auto datadir = env::env("HPCSTAT_DATADIR"); !datadir) return false;
else if (std::ifstream config_file(std::filesystem::path(*datadir) / "push.json"); !config_file)
{ fmt::print("Push failed: failed to open push.json\n"); return false; }
{ std::cout << "Push failed: failed to open push.json\n"; return false; }
else
{
auto config_string = std::string(std::istreambuf_iterator<char>(config_file), {});
@@ -29,20 +29,18 @@ namespace hpcstat::push
user_string += "::" + *std::get<3>(info);
if (users.contains(user_string))
{
auto path = fmt::format
auto path = "/api/send/message/?appToken={}&content={}&uid={}"_f
(
"/api/send/message/?appToken={}&content={}&uid={}",
token,
boost::urls::encode
(
fmt::format("{} {} {}", std::get<1>(info), std::get<0>(info), id),
"{} {} {}"_f(std::get<1>(info), std::get<0>(info), id),
boost::urls::unreserved_chars
),
users[user_string]
);
auto res = cli.Get(path.c_str());
if (res.error() != httplib::Error::Success)
{ fmt::print("Push failed: {}\n", nameof::nameof_enum(res.error())); return false; }
if (res.error() != httplib::Error::Success) { std::cout << "Push failed: {}\n"_f(res.error()); return false; }
}
}
}
@@ -52,7 +50,7 @@ namespace hpcstat::push
| ranges::views::filter([](const auto& pair)
{ return std::get<2>(pair.second) == "LNoYfq/SM7l8sFAy325WpC+li+kZl3jwST7TmP72Tz8"; })
| ranges::views::transform([](const auto& pair)
{ return fmt::format("{} {} {}", std::get<1>(pair.second), std::get<0>(pair.second), pair.first); })
{ return "{} {} {}"_f(std::get<1>(pair.second), std::get<0>(pair.second), pair.first); })
| ranges::views::chunk(20)
| ranges::views::transform([](auto chunk) { return chunk | ranges::views::join('\n'); })
| ranges::to<std::vector<std::string>>;
@@ -62,13 +60,13 @@ namespace hpcstat::push
cli.enable_server_certificate_verification(false);
for (auto& message : messages)
{
auto path = fmt::format
("/notify.php?message={}", boost::urls::encode(message, boost::urls::unreserved_chars));
auto path = "/notify.php?message={}"_f
(boost::urls::encode(message, boost::urls::unreserved_chars));
auto res = cli.Get(path.c_str());
if (res.error() != httplib::Error::Success)
{ fmt::print("Push failed: {}\n", nameof::nameof_enum(res.error())); return false; }
{ std::cout << "Push failed: {}\n"_f(res.error()); return false; }
else if (res->status != 200)
{ fmt::print("Push failed: status code {}\n", res->status); return false; }
{ std::cout << "Push failed: status code {}\n"_f(res->status); return false; }
}
}
}

29
local/pkgs/hpcstat/src/sql.cpp
View File

@@ -122,15 +122,11 @@ namespace hpcstat::sql
for (; old_data_it != old_query.end() && new_data_it != new_query.end(); ++old_data_it, ++new_data_it)
if (*old_data_it != *new_data_it)
{
std::cerr << fmt::format
("Data mismatch: {} {} != {}.\n", nameof::nameof_type<T>(), old_data_it->Id, new_data_it->Id);
return std::nullopt;
std::cerr << "Data mismatch: {} {} != {}.\n"_f(nameof::nameof_type<T>(), old_data_it->Id, new_data_it->Id);
return {};
}
if (old_data_it != old_query.end() && new_data_it == new_query.end())
{
std::cerr << fmt::format("Data mismatch in {}.\n", nameof::nameof_type<T>());
return std::nullopt;
}
{ std::cerr << "Data mismatch in {}.\n"_f(nameof::nameof_type<T>()); return {}; }
else if constexpr (requires(T data) { data.Signature; })
{
std::vector<std::tuple<std::string, std::string, std::string>> diff;
@@ -182,8 +178,7 @@ namespace hpcstat::sql
if (auto diff = job_submit.Time - submit_date; std::abs(diff) < 3600)
{
result = job_submit;
if (std::abs(diff) > 60)
std::cerr << fmt::format("large difference found: {} {}\n", job_id, diff);
if (std::abs(diff) > 60) std::cerr << "large difference found: {} {}\n"_f(job_id, diff);
break;
}
return result;
@@ -267,11 +262,10 @@ namespace hpcstat::sql
)
wks1.row(row).values() = std::vector<std::string>
{
Keys.contains(it->first) ? Keys[it->first].Username : "(unknown)",
it->first, fmt::format("{:.2f}", it->second.CpuTime),
std::to_string(it->second.LoginInteractive), std::to_string(it->second.LoginNonInteractive),
std::to_string(it->second.SubmitJob), std::to_string(it->second.FinishJobSuccess),
std::to_string(it->second.FinishJobFailed)
Keys.contains(it->first) ? Keys[it->first].Username : "(unknown)", it->first,
"{:.2f}"_f(it->second.CpuTime), "{}"_f(it->second.LoginInteractive),
"{}"_f(it->second.LoginNonInteractive), "{}"_f(it->second.SubmitJob),
"{}"_f(it->second.FinishJobSuccess), "{}"_f(it->second.FinishJobFailed)
};
doc.workbook().addWorksheet("StatisticsWithSubAccount");
auto wks2 = doc.workbook().worksheet("StatisticsWithSubAccount");
@@ -285,10 +279,9 @@ namespace hpcstat::sql
{
(Keys.contains(it->first.first) ? Keys[it->first.first].Username : "(unknown)")
+ "::" + *it->first.second,
fmt::format("{:.2f}", it->second.CpuTime),
std::to_string(it->second.LoginInteractive), std::to_string(it->second.LoginNonInteractive),
std::to_string(it->second.SubmitJob), std::to_string(it->second.FinishJobSuccess),
std::to_string(it->second.FinishJobFailed)
"{:.2f}"_f(it->second.CpuTime), "{}"_f(it->second.LoginInteractive),
"{}"_f(it->second.LoginNonInteractive), "{}"_f(it->second.SubmitJob),
"{}"_f(it->second.FinishJobSuccess), "{}"_f(it->second.FinishJobFailed)
};
doc.workbook().deleteSheet("Sheet1");
doc.save();

7
local/pkgs/hpcstat/src/ssh.cpp
View File

@@ -28,7 +28,7 @@ namespace hpcstat::ssh
++i
)
if (Keys.contains(i->str(1))) return i->str(1);
std::cerr << fmt::format("No valid fingerprint found in:\n{}\n", output.Stdout);
std::cerr << "No valid fingerprint found in:\n{}\n"_f(output.Stdout);
return std::nullopt;
}
}
@@ -44,15 +44,14 @@ namespace hpcstat::ssh
(
std::filesystem::path(*sshbindir) / "ssh-keygen",
{
"-Y", "sign", "-q",
"-f", "{}/keys/{}"_f(*sharedir, Keys[fingerprint].PubkeyFilename),
"-Y", "sign", "-q", "-f", "{}/keys/{}"_f(*sharedir, Keys[fingerprint].PubkeyFilename),
"-n", "hpcstat@chn.moe", "-"
},
message
);
!output
)
{ std::cerr << fmt::format("Failed to sign message: {}\n", message); return std::nullopt; }
{ std::cerr << "Failed to sign message: {}\n"_f(message); return {}; }
else return output.Stdout;
}
bool verify(std::string message, std::string signature, std::string fingerprint)