138 const bool serverRank = getenv(
"EON_CLIENT_STANDALONE") ==
nullptr &&
139 getenv(
"EON_SERVER_PATH") !=
nullptr;
141 const bool serverRank =
false;
143 if (argc > 1 && !serverRank) {
145 quill::Backend::stop();
156 configName =
"config_0.ini";
161 std::cerr <<
"Can't load INI file: " << configName <<
'\n'
162 <<
"problem loading parameter file, stopping\n";
174 const auto logHome = std::filesystem::current_path();
180 const bool client_standalone = !serverRank;
181 int number_of_clients;
182 if (!client_standalone) {
183 if (getenv(
"EON_NUMBER_OF_CLIENTS") ==
nullptr) {
184 QUILL_LOG_ERROR(logger,
185 "error: must set the env var EON_NUMBER_OF_CLIENTS");
189 number_of_clients = atoi(getenv(
"EON_NUMBER_OF_CLIENTS"));
191 number_of_clients = 1;
194 int eon_mpi_inited = 0;
195 MPI_Initialized(&eon_mpi_inited);
196 if (!eon_mpi_inited) {
197 int eon_mpi_provided = 0;
198 MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &eon_mpi_provided);
202 std::string config_file =
"config.ini";
203 if (client_standalone) {
205 config_file =
"config_0.ini";
207 QUILL_LOG_INFO(logger,
"Loading parameter file {}", config_file);
208 error = parameters.
load(config_file);
210 QUILL_LOG_INFO(logger,
"Loading parameter file {}",
215 QUILL_LOG_ERROR(logger,
"problem loading parameter file");
217 MPI_Abort(MPI_COMM_WORLD, 1);
221 MPI_Barrier(MPI_COMM_WORLD);
224 MPI_Comm_rank(MPI_COMM_WORLD, &irank);
226 MPI_Comm_size(MPI_COMM_WORLD, &isize);
228 std::vector<int> process_types(isize);
233 MPI_Allgather(&process_type, 1, MPI_INT, &process_types[0], 1, MPI_INT,
236 int i, servers = 0, clients = 0, potentials = 0;
237 int server_rank = -1;
238 int my_client_number = -1;
239 std::vector<int> client_ranks;
240 for (i = 0; i < isize; i++) {
241 switch (process_types[i]) {
247 my_client_number = clients;
250 client_ranks.push_back(i);
258 if (clients < number_of_clients) {
259 QUILL_LOG_ERROR(logger,
260 "didn't launch as many mpi client ranks as specified in "
261 "EON_NUMBER_OF_CLIENTS");
263 MPI_Abort(MPI_COMM_WORLD, 1);
265 clients = number_of_clients;
268 if (potentials == 0 || potentials % clients != 0) {
269 QUILL_LOG_ERROR(logger,
270 "the MPI potential needs a nonzero number of potential "
271 "ranks divisible by EON_NUMBER_OF_CLIENTS ({} potential "
272 "ranks, {} clients)",
273 potentials, clients);
275 MPI_Abort(MPI_COMM_WORLD, 1);
277 std::vector<int> potential_ranks(potentials);
279 for (i = 0, j = 0; i < isize; i++) {
280 if (process_types[i] == 2) {
281 potential_ranks[j] = i;
285 int potential_group_size = potentials / clients;
287 for (i = 0; i < clients; i++) {
288 MPI_Group orig_group, new_group;
289 MPI_Comm_group(MPI_COMM_WORLD, &orig_group);
290 int offset = i * potential_group_size;
291 MPI_Group_incl(orig_group, potential_group_size, &potential_ranks[offset],
294 MPI_Comm_create(MPI_COMM_WORLD, new_group, &pot_comm);
297 if (my_client_number < number_of_clients) {
299 potential_ranks[my_client_number * potential_group_size]);
305 for (i = 0; i < static_cast<int>(client_ranks.size()); i++) {
306 MPI_Group world_group, new_group;
307 MPI_Comm_group(MPI_COMM_WORLD, &world_group);
308 int r = client_ranks[i];
309 MPI_Group_incl(world_group, 1, &r, &new_group);
311 MPI_Comm_create(MPI_COMM_WORLD, new_group, &new_comm);
312 if (new_comm != MPI_COMM_NULL) {
313 eonc::setMpiClientComm(parameters, new_comm);
315 QUILL_LOG_INFO(logger,
"creating group with ranks: {}", r);
319 if (!client_standalone) {
320 server_rank = client_ranks[number_of_clients];
321 if (my_client_number == number_of_clients) {
322 std::ostringstream oss;
323 oss << client_ranks.at(0);
324 for (i = 1; i < number_of_clients; i++) {
325 oss <<
":" << client_ranks.at(i);
327 setenv(
"EON_CLIENT_RANKS", oss.str().c_str(), 1);
330 static_cast<wchar_t **
>(malloc(
sizeof(
wchar_t *) * 2));
331 py_argv[0] = Py_DecodeLocale(argv[0],
nullptr);
332 char *program = getenv(
"EON_SERVER_PATH");
333 py_argv[1] = Py_DecodeLocale(program,
nullptr);
334 QUILL_LOG_INFO(logger,
"rank: {} becoming {}", irank, program);
341 }
else if (my_client_number > number_of_clients) {
352 char logfilename[1024];
353 snprintf(logfilename, 1024,
"eonclient_%i.log", my_client_number);
355 auto orig_path = std::filesystem::current_path();
359 const bool keepServing = !client_standalone;
361 std::filesystem::current_path(orig_path);
362 std::string path(1024,
'\0');
364 if (!client_standalone) {
366 logger,
"client: rank {} is ready, posting send to server rank: {}!",
372 MPI_Isend(&ready, 1, MPI_INT, server_rank, 1, MPI_COMM_WORLD, &eon_rq);
373 MPI_Request_free(&eon_rq);
377 MPI_Recv(&path[0], 1024, MPI_CHAR, server_rank, 0, MPI_COMM_WORLD,
379 if (path.starts_with(
"STOPCAR")) {
380 QUILL_LOG_INFO(logger,
"rank {} got STOPCAR", irank);
384 QUILL_LOG_INFO(logger,
"client: rank: {} chdir to {}", irank, path);
386 if (
const auto chdirError =
388 QUILL_LOG_ERROR(logger,
"error: chdir: {}", *chdirError);
390 MPI_Send(&path[0], 1024, MPI_CHAR, server_rank, 0, MPI_COMM_WORLD);
395 constexpr bool keepServing =
false;
399 auto stageFailedJobLogs = [&] {
402 quill::Frontend::get_logger(std::string{
"_traceback"})) {
405 for (
const std::string_view logName :
406 {std::string_view{
"client_quill.log"},
407 std::string_view{
"client_traceback.log"}}) {
414 bool bundlingEnabled =
false;
416 if (bundleSize <= 0) {
418 bundlingEnabled =
false;
420 bundlingEnabled =
true;
423 std::vector<std::string> bundledFilenames;
424 for (
int i = 0; i < bundleSize; i++) {
427 const auto start_time = std::chrono::steady_clock::now();
429 QUILL_LOG_INFO(logger,
"Beginning Job {} of {}", i + 1, bundleSize);
430 std::vector<std::string> unbundledFilenames;
431 if (bundlingEnabled) {
437 std::string config_file =
439 QUILL_LOG_INFO(logger,
"Loading parameter file {}", config_file);
440 error = parameters.
load(config_file);
443 QUILL_LOG_ERROR(logger,
"problem loading parameter file, stopping");
446 stageFailedJobLogs();
456 std::make_unique<eonc::Parameters>(parameters), rt);
457 if (job ==
nullptr) {
458 QUILL_LOG_ERROR(logger,
"error: Unknown job: {}",
459 std::string{magic_enum::enum_name<eonc::JobType>(
463 stageFailedJobLogs();
469 std::vector<std::string> filenames;
471 filenames = job->run();
473 QUILL_LOG_CRITICAL(logger,
"[ERROR] job exited on error {}", e);
476 stageFailedJobLogs();
480 }
catch (
const std::exception &e) {
481 QUILL_LOG_CRITICAL(logger,
"[ERROR] unhandled exception: {}", e.what());
483 std::cerr <<
"[ERROR] unhandled exception: " << e.what() <<
"\n";
485 stageFailedJobLogs();
491 job->releasePotential();
494 filenames.push_back(std::string(
"_potcalls.json"));
497 auto end_time = std::chrono::steady_clock::now();
498 std::chrono::duration<double> elapsed = end_time - start_time;
500 double utime = 0, stime = 0, rtime = 0;
503 QUILL_LOG_INFO(logger,
"Timing Information:");
504 QUILL_LOG_INFO(logger,
" Real time: {:.3f} seconds", elapsed.count());
505 QUILL_LOG_INFO(logger,
" User time: {:.3f} seconds", utime);
506 QUILL_LOG_INFO(logger,
" System time: {:.3f} seconds", stime);
510 std::ofstream result_file(
"results.dat", std::ios::app);
511 if (result_file.is_open()) {
512 result_file << std::format(
"{:.12e} time_seconds\n", elapsed.count());
514 result_file << std::format(
"{:.12e} user_time\n", utime);
515 result_file << std::format(
"{:.12e} system_time\n", stime);
519 bool have_backend =
false;
521 std::ifstream prior(
"results.dat");
523 while (std::getline(prior, line)) {
524 if (line.find(
"optimizer_backend") != std::string::npos) {
531 result_file << provenance.text();
534 QUILL_LOG_ERROR(logger,
"Failed to write timing to results.dat");
540 quill::Frontend::get_logger(std::string{
"_traceback"})) {
545 constexpr std::string_view jobLogs[] = {
"client_quill.log",
546 "client_traceback.log"};
547 for (
const std::string_view logName : jobLogs) {
549 filenames.push_back(std::string{logName});
554 std::ofstream manifest(
"return_files.dat");
556 for (
const auto &fn : filenames) {
557 manifest << fn <<
"\n";
560 filenames.push_back(std::string(
"return_files.dat"));
563 if (bundlingEnabled) {
567 bundledFilenames = filenames;
572 if (client_standalone) {
577 MPI_Isend(&path[0], 1024, MPI_CHAR, server_rank, 0, MPI_COMM_WORLD,
581 MPI_Wait(&eon_rq, MPI_STATUS_IGNORE);
590 print_memory_usage();
601 quill::Backend::stop();