128 {
130
131
132
133
134
135
136
137#ifdef EONMPI
138 const bool serverRank = getenv("EON_CLIENT_STANDALONE") == nullptr &&
139 getenv("EON_SERVER_PATH") != nullptr;
140#else
141 const bool serverRank = false;
142#endif
143 if (argc > 1 && !serverRank) {
145 quill::Backend::stop();
146 return 0;
147 }
148
149
150
151 if (!serverRank) {
153
154
156 configName = "config_0.ini";
157 } else {
159 }
161 std::cerr << "Can't load INI file: " << configName << '\n'
162 << "problem loading parameter file, stopping\n";
163 return 1;
164 }
165 }
166
167
168
170 if (!logger) {
172 }
173
174 const auto logHome = std::filesystem::current_path();
175
176#ifdef EONMPI
177
178
179
180 const bool client_standalone = !serverRank;
181 int number_of_clients;
182 if (!client_standalone) {
183 if (getenv("EON_NUMBER_OF_CLIENTS") == nullptr) {
184 QUILL_LOG_ERROR(logger,
185 "error: must set the env var EON_NUMBER_OF_CLIENTS");
186 logger->flush_log();
187 return 1;
188 }
189 number_of_clients = atoi(getenv("EON_NUMBER_OF_CLIENTS"));
190 } else {
191 number_of_clients = 1;
192 }
193
194 int eon_mpi_inited = 0;
195 MPI_Initialized(&eon_mpi_inited);
196 if (!eon_mpi_inited) {
197 int eon_mpi_provided = 0;
198 MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &eon_mpi_provided);
199 }
200
202 std::string config_file = "config.ini";
203 if (client_standalone) {
205 config_file = "config_0.ini";
206 }
207 QUILL_LOG_INFO(logger, "Loading parameter file {}", config_file);
209 } else {
210 QUILL_LOG_INFO(logger, "Loading parameter file {}",
213 }
214 if (error) {
215 QUILL_LOG_ERROR(logger, "problem loading parameter file");
216 logger->flush_log();
217 MPI_Abort(MPI_COMM_WORLD, 1);
218 }
219
220
221 MPI_Barrier(MPI_COMM_WORLD);
222
223 int irank;
224 MPI_Comm_rank(MPI_COMM_WORLD, &irank);
225 int isize;
226 MPI_Comm_size(MPI_COMM_WORLD, &isize);
227
228 std::vector<int> process_types(isize);
229 int process_type;
230
231 process_type = 1;
232
233 MPI_Allgather(&process_type, 1, MPI_INT, &process_types[0], 1, MPI_INT,
234 MPI_COMM_WORLD);
235
236 int i, servers = 0, clients = 0, potentials = 0;
237 int server_rank = -1;
238 int my_client_number = -1;
239 std::vector<int> client_ranks;
240 for (i = 0; i < isize; i++) {
241 switch (process_types[i]) {
242 case 0:
243 servers++;
244 break;
245 case 1:
246 if (i == irank) {
247 my_client_number = clients;
248 }
249 clients++;
250 client_ranks.push_back(i);
251 break;
252 case 2:
253 potentials++;
254 break;
255 }
256 }
257
258 if (clients < number_of_clients) {
259 QUILL_LOG_ERROR(logger,
260 "didn't launch as many mpi client ranks as specified in "
261 "EON_NUMBER_OF_CLIENTS");
262 logger->flush_log();
263 MPI_Abort(MPI_COMM_WORLD, 1);
264 }
265 clients = number_of_clients;
266
268 if (potentials == 0 || potentials % clients != 0) {
269 QUILL_LOG_ERROR(logger,
270 "the MPI potential needs a nonzero number of potential "
271 "ranks divisible by EON_NUMBER_OF_CLIENTS ({} potential "
272 "ranks, {} clients)",
273 potentials, clients);
274 logger->flush_log();
275 MPI_Abort(MPI_COMM_WORLD, 1);
276 }
277 std::vector<int> potential_ranks(potentials);
278 int j;
279 for (i = 0, j = 0; i < isize; i++) {
280 if (process_types[i] == 2) {
281 potential_ranks[j] = i;
282 j++;
283 }
284 }
285 int potential_group_size = potentials / clients;
286
287 for (i = 0; i < clients; i++) {
288 MPI_Group orig_group, new_group;
289 MPI_Comm_group(MPI_COMM_WORLD, &orig_group);
290 int offset = i * potential_group_size;
291 MPI_Group_incl(orig_group, potential_group_size, &potential_ranks[offset],
292 &new_group);
293 MPI_Comm pot_comm;
294 MPI_Comm_create(MPI_COMM_WORLD, new_group, &pot_comm);
295 }
296
297 if (my_client_number < number_of_clients) {
299 potential_ranks[my_client_number * potential_group_size]);
300 }
301 }
302
303
305 for (i = 0; i < static_cast<int>(client_ranks.size()); i++) {
306 MPI_Group world_group, new_group;
307 MPI_Comm_group(MPI_COMM_WORLD, &world_group);
308 int r = client_ranks[i];
309 MPI_Group_incl(world_group, 1, &r, &new_group);
310 MPI_Comm new_comm;
311 MPI_Comm_create(MPI_COMM_WORLD, new_group, &new_comm);
312 if (new_comm != MPI_COMM_NULL) {
313 eonc::setMpiClientComm(parameters, new_comm);
314 }
315 QUILL_LOG_INFO(logger, "creating group with ranks: {}", r);
316 }
317 }
318
319 if (!client_standalone) {
320 server_rank = client_ranks[number_of_clients];
321 if (my_client_number == number_of_clients) {
322 std::ostringstream oss;
323 oss << client_ranks.at(0);
324 for (i = 1; i < number_of_clients; i++) {
325 oss << ":" << client_ranks.at(i);
326 }
327 setenv("EON_CLIENT_RANKS", oss.str().c_str(), 1);
328
329 wchar_t **py_argv =
330 static_cast<wchar_t **>(malloc(sizeof(wchar_t *) * 2));
331 py_argv[0] = Py_DecodeLocale(argv[0], nullptr);
332 char *program = getenv("EON_SERVER_PATH");
333 py_argv[1] = Py_DecodeLocale(program, nullptr);
334 QUILL_LOG_INFO(logger, "rank: {} becoming {}", irank, program);
335 Py_Initialize();
336 Py_Main(2, py_argv);
337 Py_FinalizeEx();
338
339 MPI_Finalize();
340 return 0;
341 } else if (my_client_number > number_of_clients) {
342 MPI_Finalize();
343 return 0;
344 }
345 }
346#endif
347
349
350#ifdef EONMPI
351
352 char logfilename[1024];
353 snprintf(logfilename, 1024, "eonclient_%i.log", my_client_number);
354
355 auto orig_path = std::filesystem::current_path();
356
357
358
359 const bool keepServing = !client_standalone;
360 while (true) {
361 std::filesystem::current_path(orig_path);
362 std::string path(1024, '\0');
363 int ready = 1;
364 if (!client_standalone) {
365 QUILL_LOG_INFO(
366 logger, "client: rank {} is ready, posting send to server rank: {}!",
367 irank, server_rank);
368
369
370 {
371 MPI_Request eon_rq;
372 MPI_Isend(&ready, 1, MPI_INT, server_rank, 1, MPI_COMM_WORLD, &eon_rq);
373 MPI_Request_free(&eon_rq);
374 }
375
376
377 MPI_Recv(&path[0], 1024, MPI_CHAR, server_rank, 0, MPI_COMM_WORLD,
378 MPI_STATUS_IGNORE);
379 if (path.starts_with("STOPCAR")) {
380 QUILL_LOG_INFO(logger, "rank {} got STOPCAR", irank);
381 MPI_Finalize();
382 return 0;
383 }
384 QUILL_LOG_INFO(logger, "client: rank: {} chdir to {}", irank, path);
385
386 if (const auto chdirError =
388 QUILL_LOG_ERROR(logger, "error: chdir: {}", *chdirError);
389 logger->flush_log();
390 MPI_Send(&path[0], 1024, MPI_CHAR, server_rank, 0, MPI_COMM_WORLD);
391 continue;
392 }
393 }
394#else
395 constexpr bool keepServing = false;
396#endif
397
398
399 auto stageFailedJobLogs = [&] {
400 logger->flush_log();
401 if (auto *trace =
402 quill::Frontend::get_logger(std::string{"_traceback"})) {
403 trace->flush_log();
404 }
405 for (const std::string_view logName :
406 {std::string_view{"client_quill.log"},
407 std::string_view{"client_traceback.log"}}) {
409 }
410 };
411
413
414 bool bundlingEnabled = false;
416 if (bundleSize <= 0) {
417 bundleSize = 1;
418 bundlingEnabled = false;
419 } else {
420 bundlingEnabled = true;
421 }
422
423 std::vector<std::string> bundledFilenames;
424 for (int i = 0; i < bundleSize; i++) {
425
426
427 const auto start_time = std::chrono::steady_clock::now();
428 if (bundleSize > 1)
429 QUILL_LOG_INFO(logger, "Beginning Job {} of {}", i + 1, bundleSize);
430 std::vector<std::string> unbundledFilenames;
431 if (bundlingEnabled) {
433 }
434
435
437 std::string config_file =
439 QUILL_LOG_INFO(logger, "Loading parameter file {}", config_file);
441
442 if (error) {
443 QUILL_LOG_ERROR(logger, "problem loading parameter file, stopping");
444 logger->flush_log();
445 if (keepServing) {
446 stageFailedJobLogs();
447 continue;
448 }
449 return 1;
450 }
451
452
453
456 std::make_unique<eonc::Parameters>(parameters), rt);
457 if (job == nullptr) {
458 QUILL_LOG_ERROR(logger, "error: Unknown job: {}",
459 std::string{magic_enum::enum_name<eonc::JobType>(
461 logger->flush_log();
462 if (keepServing) {
463 stageFailedJobLogs();
464 continue;
465 }
466 return 1;
467 }
468
469 std::vector<std::string> filenames;
470 try {
471 filenames = job->run();
472 } catch (int e) {
473 QUILL_LOG_CRITICAL(logger, "[ERROR] job exited on error {}", e);
474 logger->flush_log();
475 if (keepServing) {
476 stageFailedJobLogs();
477 continue;
478 }
479 return EXIT_FAILURE;
480 } catch (const std::exception &e) {
481 QUILL_LOG_CRITICAL(logger, "[ERROR] unhandled exception: {}", e.what());
482 logger->flush_log();
483 std::cerr << "[ERROR] unhandled exception: " << e.what() << "\n";
484 if (keepServing) {
485 stageFailedJobLogs();
486 continue;
487 }
488 return EXIT_FAILURE;
489 }
490
491 job->releasePotential();
493 job.reset();
494 filenames.push_back(std::string("_potcalls.json"));
495
496
497 auto end_time = std::chrono::steady_clock::now();
498 std::chrono::duration<double> elapsed = end_time - start_time;
499
500 double utime = 0, stime = 0, rtime = 0;
502
503 QUILL_LOG_INFO(logger, "Timing Information:");
504 QUILL_LOG_INFO(logger, " Real time: {:.3f} seconds", elapsed.count());
505 QUILL_LOG_INFO(logger, " User time: {:.3f} seconds", utime);
506 QUILL_LOG_INFO(logger, " System time: {:.3f} seconds", stime);
507
508
509
510 std::ofstream result_file("results.dat", std::ios::app);
511 if (result_file.is_open()) {
512 result_file << std::format("{:.12e} time_seconds\n", elapsed.count());
513#ifndef _WIN32
514 result_file << std::format("{:.12e} user_time\n", utime);
515 result_file << std::format("{:.12e} system_time\n", stime);
516#endif
519 bool have_backend = false;
520 {
521 std::ifstream prior("results.dat");
522 std::string line;
523 while (std::getline(prior, line)) {
524 if (line.find("optimizer_backend") != std::string::npos) {
525 have_backend = true;
526 break;
527 }
528 }
529 }
530 if (!have_backend) {
532 }
533 } else {
534 QUILL_LOG_ERROR(logger, "Failed to write timing to results.dat");
535 }
536 result_file.close();
537
538 logger->flush_log();
539 if (auto *trace =
540 quill::Frontend::get_logger(std::string{"_traceback"})) {
541 trace->flush_log();
542 }
543
544
545 constexpr std::string_view jobLogs[] = {"client_quill.log",
546 "client_traceback.log"};
547 for (const std::string_view logName : jobLogs) {
549 filenames.push_back(std::string{logName});
550 }
551 }
552
553 {
554 std::ofstream manifest("return_files.dat");
555 if (manifest) {
556 for (const auto &fn : filenames) {
557 manifest << fn << "\n";
558 }
559 }
560 filenames.push_back(std::string("return_files.dat"));
561 }
562
563 if (bundlingEnabled) {
566 } else {
567 bundledFilenames = filenames;
568 }
569 }
570
571#ifdef EONMPI
572 if (client_standalone) {
573 break;
574 }
575 {
576 MPI_Request eon_rq;
577 MPI_Isend(&path[0], 1024, MPI_CHAR, server_rank, 0, MPI_COMM_WORLD,
578 &eon_rq);
579
580
581 MPI_Wait(&eon_rq, MPI_STATUS_IGNORE);
582 }
583
584
585 }
586#endif
587
588#ifdef OSX
589#ifndef __aarch64__
590 print_memory_usage();
591#endif
592#endif
593
594#ifdef EONMPI
595
596
597 MPI_Finalize();
598#endif
599
600
601 quill::Backend::stop();
602 return EXIT_SUCCESS;
603}
void set_mpi_potential_rank(int rank)
const potential_options_t & potential_options() const
const main_options_t & main_options() const
int load(std::string_view filename)
void write_summary(const std::string &path="_potcalls.json") const
Move-only composition root for process resources (dlopen loaders and the potential registry).
PotRegistry & pots() noexcept
std::string getRelevantFile(std::string filename)
bool stageReturnLog(std::string_view logHome, std::string_view name)
Copy name from logHome into the current directory when that file is not already this directory's copy...
std::optional< std::string > enterJobDirectory(std::string_view jobPath)
Enter jobPath as the working directory.
std::unique_ptr< Job > makeJob(std::unique_ptr< Parameters > params, Runtime &runtime)
Borrow: caller keeps Runtime alive.
void getTime(double *real, double *user, double *sys)
bool existsFile(std::string filename)
quill::Logger * init_client() noexcept
Start the client backend and the combi / traceback loggers once.
quill::Logger * get() noexcept
Get or create the default "combi" logger.
const std::string & provenance()
Stamp text from the last successful eindir bind. Empty when unused.
void bundle(int number, const std::vector< std::string > &filenames, std::vector< std::string > *bundledFilenames)
void deleteUnbundledFiles(const std::vector< std::string > &unbundledFilenames)
std::vector< std::string > unbundle(int number)
void commandLine(int argc, char **argv)
JobResultProvenance provenanceForJob(const Parameters ¶ms)
Optimizer backend, engine identity, rgpot pin, and xts ABI from a job.
Optimizer, xtsci, eindir, and rgpot identity carried on a JobResult.