Loading...
Searching...
No Matches
ClientEON.cpp File Reference
#include "eon/EonLogger.h"
#include "eon/BaseStructures.h"
#include "eon/Bundling.h"
#include "eon/CommandLine.h"
#include "eon/EpiCenters.h"
#include "eon/HelperFunctions.h"
#include "eon/Job.h"
#include "eon/JobResult.h"
#include "eon/Parameters.h"
#include "eon/Potential.h"
#include "eon/Runtime.h"
#include "version.h"
#include <cstdlib>
#include <exception>
#include <format>
#include <fstream>
#include <iostream>
#include <string_view>
#include <cerrno>
#include <chrono>
#include <cstring>
#include <ctime>
#include <filesystem>
#include "eon/fpe_handler.h"
#include <sys/resource.h>
#include <sys/time.h>
#include <sys/utsname.h>
#include <unistd.h>

Go to the source code of this file.

Functions

void printSystemInfo ()
static int eonClientMain (int argc, char **argv)
int main (int argc, char **argv)

Function Documentation

◆ eonClientMain()

int eonClientMain ( int argc,
char ** argv )
static

Definition at line 128 of file ClientEON.cpp.

128 {
129 eonc::Parameters parameters;
130
131 // Help, version, and one-shot argv jobs must not pay logger setup first.
132 // commandLine starts the backend only after the flag parse commits to work.
133 // In an MPI build only a rank an eOn server launched (EON_SERVER_PATH set,
134 // EON_CLIENT_STANDALONE unset) skips the parse: the server passes it no
135 // flags. A standalone or hand-started MPI client reads its flags like the
136 // serial client; rgpot initialises MPI if a calculator group needs it.
137#ifdef EONMPI
138 const bool serverRank = getenv("EON_CLIENT_STANDALONE") == nullptr &&
139 getenv("EON_SERVER_PATH") != nullptr;
140#else
141 const bool serverRank = false;
142#endif
143 if (argc > 1 && !serverRank) {
144 eonc::commandLine(argc, argv);
145 quill::Backend::stop();
146 return 0;
147 }
148
149 // A missing parameter file is not a job. Reject it before the log
150 // thread and the file sinks. Help and version already returned above.
151 if (!serverRank) {
152 std::string configName = parameters.main_options().iniFilename;
153 // A bundle is config_0.ini plus its siblings. That set is a parameter
154 // file even when config.ini itself is absent.
155 if (eonc::helpers::existsFile("config_0.ini")) {
156 configName = "config_0.ini";
157 } else {
158 configName = eonc::helpers::getRelevantFile(configName);
159 }
160 if (!eonc::helpers::existsFile(configName)) {
161 std::cerr << "Can't load INI file: " << configName << '\n'
162 << "problem loading parameter file, stopping\n";
163 return 1;
164 }
165 }
166
167 // Quill backend, file sinks, and the combi logger. Deferred until a job
168 // path actually runs so process start is not dominated by the log thread.
169 auto *logger = eonc::log::init_client();
170 if (!logger) {
171 logger = eonc::log::get();
172 }
173 // File sinks open relative to this directory. MPI jobs chdir later.
174 const auto logHome = std::filesystem::current_path();
175
176#ifdef EONMPI
177 // The same rule as the flag parse above: only a rank an eOn server
178 // launched (EON_SERVER_PATH set, EON_CLIENT_STANDALONE unset) runs as a
179 // server-driven client. Any other MPI client runs config.ini standalone.
180 const bool client_standalone = !serverRank;
181 int number_of_clients;
182 if (!client_standalone) {
183 if (getenv("EON_NUMBER_OF_CLIENTS") == nullptr) {
184 QUILL_LOG_ERROR(logger,
185 "error: must set the env var EON_NUMBER_OF_CLIENTS");
186 logger->flush_log();
187 return 1;
188 }
189 number_of_clients = atoi(getenv("EON_NUMBER_OF_CLIENTS"));
190 } else {
191 number_of_clients = 1;
192 }
193
194 int eon_mpi_inited = 0;
195 MPI_Initialized(&eon_mpi_inited);
196 if (!eon_mpi_inited) {
197 int eon_mpi_provided = 0;
198 MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &eon_mpi_provided);
199 }
200
201 int error;
202 std::string config_file = "config.ini";
203 if (client_standalone) {
204 if (eonc::helpers::existsFile("config_0.ini")) {
205 config_file = "config_0.ini";
206 }
207 QUILL_LOG_INFO(logger, "Loading parameter file {}", config_file);
208 error = parameters.load(config_file);
209 } else {
210 QUILL_LOG_INFO(logger, "Loading parameter file {}",
211 parameters.main_options().iniFilename);
212 error = parameters.load(parameters.main_options().iniFilename);
213 }
214 if (error) {
215 QUILL_LOG_ERROR(logger, "problem loading parameter file");
216 logger->flush_log();
217 MPI_Abort(MPI_COMM_WORLD, 1);
218 }
219
220 // All ranks must have loaded Parameters before the process-type Allgather.
221 MPI_Barrier(MPI_COMM_WORLD);
222
223 int irank;
224 MPI_Comm_rank(MPI_COMM_WORLD, &irank);
225 int isize;
226 MPI_Comm_size(MPI_COMM_WORLD, &isize);
227
228 std::vector<int> process_types(isize);
229 int process_type;
230
231 process_type = 1;
232
233 MPI_Allgather(&process_type, 1, MPI_INT, &process_types[0], 1, MPI_INT,
234 MPI_COMM_WORLD);
235
236 int i, servers = 0, clients = 0, potentials = 0;
237 int server_rank = -1;
238 int my_client_number = -1;
239 std::vector<int> client_ranks;
240 for (i = 0; i < isize; i++) {
241 switch (process_types[i]) {
242 case 0:
243 servers++;
244 break;
245 case 1:
246 if (i == irank) {
247 my_client_number = clients;
248 }
249 clients++;
250 client_ranks.push_back(i);
251 break;
252 case 2:
253 potentials++;
254 break;
255 }
256 }
257
258 if (clients < number_of_clients) {
259 QUILL_LOG_ERROR(logger,
260 "didn't launch as many mpi client ranks as specified in "
261 "EON_NUMBER_OF_CLIENTS");
262 logger->flush_log();
263 MPI_Abort(MPI_COMM_WORLD, 1);
264 }
265 clients = number_of_clients;
266
267 if (parameters.potential_options().potential == eonc::PotType::MPI) {
268 if (potentials == 0 || potentials % clients != 0) {
269 QUILL_LOG_ERROR(logger,
270 "the MPI potential needs a nonzero number of potential "
271 "ranks divisible by EON_NUMBER_OF_CLIENTS ({} potential "
272 "ranks, {} clients)",
273 potentials, clients);
274 logger->flush_log();
275 MPI_Abort(MPI_COMM_WORLD, 1);
276 }
277 std::vector<int> potential_ranks(potentials);
278 int j;
279 for (i = 0, j = 0; i < isize; i++) {
280 if (process_types[i] == 2) {
281 potential_ranks[j] = i;
282 j++;
283 }
284 }
285 int potential_group_size = potentials / clients;
286
287 for (i = 0; i < clients; i++) {
288 MPI_Group orig_group, new_group;
289 MPI_Comm_group(MPI_COMM_WORLD, &orig_group);
290 int offset = i * potential_group_size;
291 MPI_Group_incl(orig_group, potential_group_size, &potential_ranks[offset],
292 &new_group);
293 MPI_Comm pot_comm;
294 MPI_Comm_create(MPI_COMM_WORLD, new_group, &pot_comm);
295 }
296
297 if (my_client_number < number_of_clients) {
298 parameters.set_mpi_potential_rank(
299 potential_ranks[my_client_number * potential_group_size]);
300 }
301 }
302
303 // LAMMPS MPI communicator setup (runtime check, not compile-time)
305 for (i = 0; i < static_cast<int>(client_ranks.size()); i++) {
306 MPI_Group world_group, new_group;
307 MPI_Comm_group(MPI_COMM_WORLD, &world_group);
308 int r = client_ranks[i];
309 MPI_Group_incl(world_group, 1, &r, &new_group);
310 MPI_Comm new_comm;
311 MPI_Comm_create(MPI_COMM_WORLD, new_group, &new_comm);
312 if (new_comm != MPI_COMM_NULL) {
313 eonc::setMpiClientComm(parameters, new_comm);
314 }
315 QUILL_LOG_INFO(logger, "creating group with ranks: {}", r);
316 }
317 }
318
319 if (!client_standalone) {
320 server_rank = client_ranks[number_of_clients];
321 if (my_client_number == number_of_clients) {
322 std::ostringstream oss;
323 oss << client_ranks.at(0);
324 for (i = 1; i < number_of_clients; i++) {
325 oss << ":" << client_ranks.at(i);
326 }
327 setenv("EON_CLIENT_RANKS", oss.str().c_str(), 1);
328
329 wchar_t **py_argv =
330 static_cast<wchar_t **>(malloc(sizeof(wchar_t *) * 2));
331 py_argv[0] = Py_DecodeLocale(argv[0], nullptr);
332 char *program = getenv("EON_SERVER_PATH");
333 py_argv[1] = Py_DecodeLocale(program, nullptr);
334 QUILL_LOG_INFO(logger, "rank: {} becoming {}", irank, program);
335 Py_Initialize();
336 Py_Main(2, py_argv);
337 Py_FinalizeEx();
338 // GH
339 MPI_Finalize();
340 return 0;
341 } else if (my_client_number > number_of_clients) {
342 MPI_Finalize();
343 return 0;
344 }
345 }
346#endif
347
348 eonc::enableFPE(); // from ExceptionsEON.h
349
350#ifdef EONMPI
351 // Server sends a path starting with STOPCAR to end this loop.
352 char logfilename[1024];
353 snprintf(logfilename, 1024, "eonclient_%i.log", my_client_number);
354
355 auto orig_path = std::filesystem::current_path();
356 // In client/server mode a failed job is logged and its directory handed
357 // back without results.dat; the server skips it and this rank stays in
358 // the pool. A standalone rank still exits with the error.
359 const bool keepServing = !client_standalone;
360 while (true) {
361 std::filesystem::current_path(orig_path);
362 std::string path(1024, '\0');
363 int ready = 1;
364 if (!client_standalone) {
365 QUILL_LOG_INFO(
366 logger, "client: rank {} is ready, posting send to server rank: {}!",
367 irank, server_rank);
368 // Tag "1" is to interrupt the main loop and tell the communicator that a
369 // client is ready
370 {
371 MPI_Request eon_rq;
372 MPI_Isend(&ready, 1, MPI_INT, server_rank, 1, MPI_COMM_WORLD, &eon_rq);
373 MPI_Request_free(&eon_rq);
374 }
375
376 // Get the path we should run in from the server
377 MPI_Recv(&path[0], 1024, MPI_CHAR, server_rank, 0, MPI_COMM_WORLD,
378 MPI_STATUS_IGNORE);
379 if (path.starts_with("STOPCAR")) {
380 QUILL_LOG_INFO(logger, "rank {} got STOPCAR", irank);
381 MPI_Finalize();
382 return 0;
383 }
384 QUILL_LOG_INFO(logger, "client: rank: {} chdir to {}", irank, path);
385
386 if (const auto chdirError =
387 eonc::helpers::enterJobDirectory(path.c_str())) {
388 QUILL_LOG_ERROR(logger, "error: chdir: {}", *chdirError);
389 logger->flush_log();
390 MPI_Send(&path[0], 1024, MPI_CHAR, server_rank, 0, MPI_COMM_WORLD);
391 continue;
392 }
393 }
394#else
395 constexpr bool keepServing = false;
396#endif
397 // Flushes the logs into the job directory so a failed job still shows
398 // its error to whoever reads the returned directory.
399 auto stageFailedJobLogs = [&] {
400 logger->flush_log();
401 if (auto *trace =
402 quill::Frontend::get_logger(std::string{"_traceback"})) {
403 trace->flush_log();
404 }
405 for (const std::string_view logName :
406 {std::string_view{"client_quill.log"},
407 std::string_view{"client_traceback.log"}}) {
408 eonc::helpers::stageReturnLog(logHome.string(), logName);
409 }
410 };
411
413
414 bool bundlingEnabled = false;
415 int bundleSize = eonc::getBundleSize();
416 if (bundleSize <= 0) {
417 bundleSize = 1;
418 bundlingEnabled = false;
419 } else {
420 bundlingEnabled = true;
421 }
422
423 std::vector<std::string> bundledFilenames;
424 for (int i = 0; i < bundleSize; i++) {
425 // This job only. A clock above the bundle or MPI loop also counts
426 // earlier jobs and the idle wait between them.
427 const auto start_time = std::chrono::steady_clock::now();
428 if (bundleSize > 1)
429 QUILL_LOG_INFO(logger, "Beginning Job {} of {}", i + 1, bundleSize);
430 std::vector<std::string> unbundledFilenames;
431 if (bundlingEnabled) {
432 unbundledFilenames = eonc::unbundle(i);
433 }
434
435 // check to see if parameters file exists before loading
436 int error = 0;
437 std::string config_file =
439 QUILL_LOG_INFO(logger, "Loading parameter file {}", config_file);
440 error = parameters.load(config_file);
441
442 if (error) {
443 QUILL_LOG_ERROR(logger, "problem loading parameter file, stopping");
444 logger->flush_log();
445 if (keepServing) {
446 stageFailedJobLogs();
447 continue;
448 }
449 return 1;
450 }
451
452 // Determine what type of job we are running according to the parameters
453 // file.
454 eonc::Runtime rt;
455 auto job = eonc::helpers::makeJob(
456 std::make_unique<eonc::Parameters>(parameters), rt);
457 if (job == nullptr) {
458 QUILL_LOG_ERROR(logger, "error: Unknown job: {}",
459 std::string{magic_enum::enum_name<eonc::JobType>(
460 parameters.main_options().job)});
461 logger->flush_log();
462 if (keepServing) {
463 stageFailedJobLogs();
464 continue;
465 }
466 return 1;
467 }
468
469 std::vector<std::string> filenames;
470 try {
471 filenames = job->run();
472 } catch (int e) {
473 QUILL_LOG_CRITICAL(logger, "[ERROR] job exited on error {}", e);
474 logger->flush_log();
475 if (keepServing) {
476 stageFailedJobLogs();
477 continue;
478 }
479 return EXIT_FAILURE;
480 } catch (const std::exception &e) {
481 QUILL_LOG_CRITICAL(logger, "[ERROR] unhandled exception: {}", e.what());
482 logger->flush_log();
483 std::cerr << "[ERROR] unhandled exception: " << e.what() << "\n";
484 if (keepServing) {
485 stageFailedJobLogs();
486 continue;
487 }
488 return EXIT_FAILURE;
489 }
490
491 job->releasePotential();
492 rt.pots().write_summary();
493 job.reset();
494 filenames.push_back(std::string("_potcalls.json"));
495
496 // Finalize Timing Information
497 auto end_time = std::chrono::steady_clock::now();
498 std::chrono::duration<double> elapsed = end_time - start_time;
499
500 double utime = 0, stime = 0, rtime = 0;
501 eonc::helpers::getTime(&rtime, &utime, &stime);
502
503 QUILL_LOG_INFO(logger, "Timing Information:");
504 QUILL_LOG_INFO(logger, " Real time: {:.3f} seconds", elapsed.count());
505 QUILL_LOG_INFO(logger, " User time: {:.3f} seconds", utime);
506 QUILL_LOG_INFO(logger, " System time: {:.3f} seconds", stime);
507
508 // results.dat contract is "<value> <key>" (same as all job writers and
509 // eon.fileio.parse_results / eon_schema.jobs adapters).
510 std::ofstream result_file("results.dat", std::ios::app);
511 if (result_file.is_open()) {
512 result_file << std::format("{:.12e} time_seconds\n", elapsed.count());
513#ifndef _WIN32
514 result_file << std::format("{:.12e} user_time\n", utime);
515 result_file << std::format("{:.12e} system_time\n", stime);
516#endif
518 eonc::provenanceForJob(parameters);
519 bool have_backend = false;
520 {
521 std::ifstream prior("results.dat");
522 std::string line;
523 while (std::getline(prior, line)) {
524 if (line.find("optimizer_backend") != std::string::npos) {
525 have_backend = true;
526 break;
527 }
528 }
529 }
530 if (!have_backend) {
531 result_file << provenance.text();
532 }
533 } else {
534 QUILL_LOG_ERROR(logger, "Failed to write timing to results.dat");
535 }
536 result_file.close();
537
538 logger->flush_log();
539 if (auto *trace =
540 quill::Frontend::get_logger(std::string{"_traceback"})) {
541 trace->flush_log();
542 }
543 // Quill opens these before an MPI job chdir. Copy them into the job
544 // directory so return_files.dat does not name a missing log.
545 constexpr std::string_view jobLogs[] = {"client_quill.log",
546 "client_traceback.log"};
547 for (const std::string_view logName : jobLogs) {
548 if (eonc::helpers::stageReturnLog(logHome.string(), logName)) {
549 filenames.push_back(std::string{logName});
550 }
551 }
552
553 {
554 std::ofstream manifest("return_files.dat");
555 if (manifest) {
556 for (const auto &fn : filenames) {
557 manifest << fn << "\n";
558 }
559 }
560 filenames.push_back(std::string("return_files.dat"));
561 }
562
563 if (bundlingEnabled) {
564 eonc::bundle(i, filenames, &bundledFilenames);
565 eonc::deleteUnbundledFiles(unbundledFilenames);
566 } else {
567 bundledFilenames = filenames;
568 }
569 }
570
571#ifdef EONMPI
572 if (client_standalone) {
573 break;
574 }
575 {
576 MPI_Request eon_rq;
577 MPI_Isend(&path[0], 1024, MPI_CHAR, server_rank, 0, MPI_COMM_WORLD,
578 &eon_rq);
579 // path is destroyed at the end of this iteration. Request_free does
580 // not complete the send, so the buffer stays live until Wait returns.
581 MPI_Wait(&eon_rq, MPI_STATUS_IGNORE);
582 }
583
584 // End of MPI while loop
585 }
586#endif
587
588#ifdef OSX
589#ifndef __aarch64__
590 print_memory_usage();
591#endif
592#endif
593
594#ifdef EONMPI
595 // Clean shutdown for both the standalone single-rank case and the
596 // client/server case; the standalone path previously aborted MPI_COMM_WORLD.
597 MPI_Finalize();
598#endif
599
600 // Ensure all queued log messages are flushed before exiting
601 quill::Backend::stop();
602 return EXIT_SUCCESS;
603}
void printSystemInfo()
Definition ClientEON.cpp:95
void set_mpi_potential_rank(int rank)
const potential_options_t & potential_options() const
const main_options_t & main_options() const
int load(std::string_view filename)
void write_summary(const std::string &path="_potcalls.json") const
Move-only composition root for process resources (dlopen loaders and the potential registry).
Definition Runtime.h:27
PotRegistry & pots() noexcept
Definition Runtime.cpp:33
std::string error()
Definition DynLib.h:82
std::string getRelevantFile(std::string filename)
bool stageReturnLog(std::string_view logHome, std::string_view name)
Copy name from logHome into the current directory when that file is not already this directory's copy...
std::optional< std::string > enterJobDirectory(std::string_view jobPath)
Enter jobPath as the working directory.
std::unique_ptr< Job > makeJob(std::unique_ptr< Parameters > params, Runtime &runtime)
Borrow: caller keeps Runtime alive.
void getTime(double *real, double *user, double *sys)
bool existsFile(std::string filename)
quill::Logger * init_client() noexcept
Start the client backend and the combi / traceback loggers once.
Definition EonLogger.h:43
quill::Logger * get() noexcept
Get or create the default "combi" logger.
Definition EonLogger.h:111
const std::string & provenance()
Stamp text from the last successful eindir bind. Empty when unused.
void bundle(int number, const std::vector< std::string > &filenames, std::vector< std::string > *bundledFilenames)
Definition Bundling.cpp:180
void deleteUnbundledFiles(const std::vector< std::string > &unbundledFilenames)
Definition Bundling.cpp:145
std::vector< std::string > unbundle(int number)
Definition Bundling.cpp:71
int getBundleSize()
Definition Bundling.cpp:27
void commandLine(int argc, char **argv)
JobResultProvenance provenanceForJob(const Parameters &params)
Optimizer backend, engine identity, rgpot pin, and xts ABI from a job.
void enableFPE()
Optimizer, xtsci, eindir, and rgpot identity carried on a JobResult.
Definition JobResult.h:35

◆ main()

int main ( int argc,
char ** argv )

Definition at line 624 of file ClientEON.cpp.

624 {
625 try {
626 return eonClientMain(argc, argv);
627 } catch (const std::exception &e) {
628 return reportFatal(e.what());
629 } catch (...) {
630 return reportFatal("exception of unknown type");
631 }
632}
static int eonClientMain(int argc, char **argv)

◆ printSystemInfo()

void printSystemInfo ( )

Definition at line 95 of file ClientEON.cpp.

95 {
96 auto *log = eonc::log::get();
97 QUILL_LOG_INFO(log, "eOn Client");
98 QUILL_LOG_INFO(log, "{}", VERSION_STRING);
99#ifndef __aarch64__
100 QUILL_LOG_INFO(log, "OS: {}", OS_INFO);
101 QUILL_LOG_INFO(log, "Arch: {}", ARCH);
102#endif
103
104#ifdef _WIN32
105 TCHAR hostname[MAX_COMPUTERNAME_LENGTH + 1];
106 DWORD size = sizeof(hostname) / sizeof(hostname[0]);
107 if (GetComputerName(hostname, &size)) {
108 QUILL_LOG_INFO(log, "Hostname: {}", hostname);
109 } else {
110 QUILL_LOG_ERROR(log, "Failed to get hostname");
111 }
112 QUILL_LOG_INFO(log, "PID: {}", GetCurrentProcessId());
113#else
114 struct utsname systemInfo;
115 int status = uname(&systemInfo);
116 if (status == 0) {
117 QUILL_LOG_INFO(log, "Hostname: {}", systemInfo.nodename);
118 QUILL_LOG_INFO(log, "PID: {}", getpid());
119 } else {
120 QUILL_LOG_ERROR(log, "Failed to get system information");
121 }
122#endif
123
124 std::filesystem::path cwd = std::filesystem::current_path();
125 QUILL_LOG_INFO(log, "DIR: {}", cwd.string());
126}