chn/nixos
1
0
Fork 0
mirror of https://github.com/CHN-beta/nixos.git synced 2024-10-22 21:18:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

localPackages.hpcstat: export job detail to excel

Browse Source
This commit is contained in:
陈浩南 2024-06-18 19:54:37 +08:00
parent 37e6d70561
commit 173dd4a3d7
6 changed files with 40 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
local/pkgs/default.nix
View File

@ -73,7 +73,7 @@ inputs: rec
{ src = inputs.topInputs.kylin-virtual-keyboard; };
biu = inputs.pkgs.callPackage ./biu { inherit nameof zpp-bits; };
zxorm = inputs.pkgs.callPackage ./zxorm { src = inputs.topInputs.zxorm; };
hpcstat = inputs.pkgs.callPackage ./hpcstat { inherit nameof sqlite-orm zpp-bits date biu; };
hpcstat = inputs.pkgs.callPackage ./hpcstat { inherit nameof sqlite-orm zpp-bits date biu openxlsx; };
openxlsx = inputs.pkgs.callPackage ./openxlsx { src = inputs.topInputs.openxlsx; };
sqlite-orm = inputs.pkgs.callPackage ./sqlite-orm { src = inputs.topInputs.sqlite-orm; };
mkPnpmPackage = inputs.pkgs.callPackage ./mkPnpmPackage.nix {};

3
local/pkgs/hpcstat/CMakeLists.txt
View File

@ -21,6 +21,7 @@ find_package(date REQUIRED)
find_package(httplib REQUIRED)
find_package(termcolor REQUIRED)
find_package(biu REQUIRED)
find_package(OpenXLSX REQUIRED)
add_executable(hpcstat src/main.cpp src/env.cpp src/keys.cpp src/ssh.cpp src/sql.cpp src/lfs.cpp src/common.cpp
src/push.cpp src/disk.cpp)
@ -28,7 +29,7 @@ target_compile_features(hpcstat PRIVATE cxx_std_23)
target_include_directories(hpcstat PRIVATE ${PROJECT_SOURCE_DIR}/include ${ZPP_BITS_INCLUDE_DIR})
target_link_libraries(hpcstat PRIVATE Boost::headers Boost::filesystem sqlite_orm::sqlite_orm
nlohmann_json::nlohmann_json range-v3::range-v3 date::date date::date-tz httplib::httplib
termcolor::termcolor biu::biu)
termcolor::termcolor biu::biu OpenXLSX::OpenXLSX)
target_compile_definitions(hpcstat PRIVATE HPCSTAT_VERSION="${HPCSTAT_VERSION}")
install(TARGETS hpcstat RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})

4
local/pkgs/hpcstat/default.nix
View File

@ -1,13 +1,13 @@
{
stdenv, cmake, pkg-config, standalone ? false, version ? null, makeWrapper, lib,
boost, fmt, sqlite-orm, nlohmann_json, zpp-bits, range-v3, nameof, openssh, sqlite, date, httplib, openssl,
boost, fmt, sqlite-orm, nlohmann_json, zpp-bits, range-v3, nameof, openssh, sqlite, date, httplib, openssl, openxlsx,
termcolor, duc, biu
}: stdenv.mkDerivation
{
name = "hpcstat";
src = ./.;
buildInputs =
[ boost fmt sqlite-orm nlohmann_json zpp-bits range-v3 nameof sqlite date httplib termcolor openssl biu ];
[ boost fmt sqlite-orm nlohmann_json zpp-bits range-v3 nameof sqlite date httplib termcolor openssl biu openxlsx ];
nativeBuildInputs = [ cmake pkg-config makeWrapper ];
cmakeFlags = lib.optionals (version != null) [ "-DHPCSTAT_VERSION=${version}" ];
postInstall =

2
local/pkgs/hpcstat/include/hpcstat/sql.hpp
View File

@ -63,7 +63,7 @@ namespace hpcstat::sql
std::optional<std::vector<std::tuple<std::string, std::string, std::string>>>
verify(std::string old_db, std::string new_db);
// 将某个月份的数据导出
bool export_data(long start_time, long end_time);
bool export_data(long start_time, long end_time, std::string filename);
// 检查任务状态，返回有变化的任务 id、名称、现在的状态、提交时的 key、subaccount
// 如果没有找到提交时的信息，则忽略这个任务
std::optional<std::map<unsigned, std::tuple<std::string, std::string, std::string, std::optional<std::string>>>>

3
local/pkgs/hpcstat/src/main.cpp
View File

@ -167,8 +167,7 @@ int main(int argc, const char** argv)
auto end = sys_seconds(sys_days(month(month_n) / 1 / year_n + months(1)))
.time_since_epoch().count();
lock.lock();
if (!sql::export_data(begin, end))
return 1;
if (!sql::export_data(begin, end, "{}{}.xlsx"_f(year_n, month_n))) return 1;
}
else if (args[1] == "push")
{

50
local/pkgs/hpcstat/src/sql.cpp
View File

@ -7,6 +7,7 @@
# include <nameof.hpp>
# define SQLITE_ORM_OPTIONAL_SUPPORTED
# include <sqlite_orm/sqlite_orm.h>
# include <OpenXLSX.hpp>
namespace hpcstat::sql
{
@ -182,7 +183,7 @@ namespace hpcstat::sql
}
return result;
}
bool export_data(long start_time, long end_time)
bool export_data(long start_time, long end_time, std::string filename)
{
if (auto conn = connect(); !conn) return false;
else
@ -201,8 +202,8 @@ namespace hpcstat::sql
struct StatJob
{
unsigned JobId;
std::optional<std::string> Key, SessionId, SubmitDir, JobCommand, Ip;
std::string JobResult, SubmitTime;
std::optional<std::string> Key, SubmitDir, JobCommand;
std::string JobResult, SubmitTime, JobDetail;
double CpuTime;
};
std::vector<StatJob> stat_job;
@ -214,17 +215,18 @@ namespace hpcstat::sql
)
{
stat_job.push_back
({ .JobId = it.JobId, .JobResult = it.JobResult, .SubmitTime = it.SubmitTime, .CpuTime = it.CpuTime });
({
.JobId = it.JobId, .JobResult = it.JobResult, .SubmitTime = it.SubmitTime, .JobDetail = it.JobDetail,
.CpuTime = it.CpuTime
});
if (auto job_in_submit = search_job_in_submit
(conn, it.JobId, it.SubmitTime))
{
{
auto& _ = stat_job.back();
_.Key = job_in_submit->Key;
_.SessionId = job_in_submit->SessionId;
_.SubmitDir = job_in_submit->SubmitDir;
_.JobCommand = job_in_submit->JobCommand;
_.Ip = job_in_submit->Ip;
}
stat_account[job_in_submit->Key].CpuTime += it.CpuTime / 3600;
if (it.JobResult == "DONE") stat_account[job_in_submit->Key].FinishJobSuccess++;
@ -260,8 +262,8 @@ namespace hpcstat::sql
}
}
// export to markdown
std::cout << "| 账号 | 使用核时 | 登陆次数(交互式) | 登陆次数(非交互式) | 成功任务数 | 失败任务数 | SSH密钥编号::指纹 |\n";
std::cout << "| :--: | :--: | :--: | :--: | :--: | :--: | :--: |\n";
std::cout << "| 账号 | 使用核时 | 登陆次数(总计/交互式/非交互式) | 完成任务(总计/成功/失败) | SSH密钥编号::指纹 |\n";
std::cout << "| :--: | :--: | :--: | :--: | :--: |\n";
std::vector<std::pair<std::optional<std::string>, StatAccount>> stat_account_vector
(stat_account.begin(), stat_account.end());
auto compare = [](auto& a, auto& b)
@ -272,10 +274,12 @@ namespace hpcstat::sql
};
std::sort(stat_account_vector.begin(), stat_account_vector.end(), compare);
for (auto& [key, stat] : stat_account_vector)
std::cout << "| {} | {:.2f} | {} | {} | {} | {} | `{}::{}` |\n"_f
std::cout << "| {} | {:.2f} | {}/{}/{} | {}/{}/{} | `{}` |\n"_f
(
key ? Keys[*key].Username : "(unknown)", stat.CpuTime, stat.LoginInteractive, stat.LoginNonInteractive,
stat.FinishJobSuccess, stat.FinishJobFailed, key ? Keys[*key].PubkeyFilename : "", key
key ? Keys[*key].Username : "(unknown)", stat.CpuTime,
stat.LoginInteractive + stat.LoginNonInteractive, stat.LoginInteractive, stat.LoginNonInteractive,
stat.FinishJobSuccess + stat.FinishJobFailed, stat.FinishJobSuccess, stat.FinishJobFailed,
key ? "{}::SHA256:{}"_f(Keys[*key].PubkeyFilename, *key) : "(unknown)"
);
for (auto& [key_subaccount, stat] : stat_subaccount)
std::cout << "| {}::{} | {:.2f} | {} | {} | {} | {} | `{}::{}` |\n"_f
@ -284,12 +288,24 @@ namespace hpcstat::sql
stat.LoginInteractive, stat.LoginNonInteractive, stat.FinishJobSuccess, stat.FinishJobFailed,
Keys[key_subaccount.first].PubkeyFilename, key_subaccount.first
);
std::cout << "\n";
std::cout << "| 任务ID | 任务结果 | 提交时间 | 使用核时 | SSH指纹 | 会话ID | 提交目录 | 任务命令 | TCP连接 |\n";
std::cout << "| :--: | :--: | :--: | :--: | :--: | :--: | :--: | :--: | :--: |\n";
for (auto& it : stat_job)
std::cout << "| {} | {} | {} | {:.2f} | `{}` | `{}` | `{}` | `{}` | `{}` |\n"_f
(it.JobId, it.JobResult, it.SubmitTime, it.CpuTime, it.Key, it.SessionId, it.SubmitDir, it.JobCommand, it.Ip);
// export to excel
OpenXLSX::XLDocument doc;
doc.create(filename);
auto wks1 = doc.workbook().worksheet("Sheet1");
wks1.row(1).values() = std::vector<std::string>
{
"用户", "任务ID", "结果", "核时", "提交时间", "提交时当前目录", "提交命令",
"详情"
};
for (auto [row, it] = std::tuple(2, stat_job.begin()); it != stat_job.end(); it++, row++)
wks1.row(row).values() = std::vector<std::string>
{
Keys.contains(*it->Key) ? Keys[*it->Key].Username : "(unknown)",
"{}"_f(it->JobId), it->JobResult, "{:.2f}"_f(it->CpuTime), it->SubmitTime,
it->SubmitDir.value_or("(unknown)"), it->JobCommand.value_or("(unknown)"),
it->JobDetail
};
doc.save();
return true;
}
}