[docs]definitialize_ray(cfg=None,force=False):ifray.is_initialized()andnotforce:returnfromray.runtime_envimportRuntimeEnvifcfgisNone:ray_address="auto"logger.warning("No ray config provided, using default ray address 'auto'.")else:ray_address=cfg.ray_addressruntime_env=RuntimeEnv(env_vars={RAY_JOB_ENV_VAR:os.environ.get(RAY_JOB_ENV_VAR,"0")})ray.init(ray_address,ignore_reinit_error=True,runtime_env=runtime_env)
[docs]defray_available_memories():"""Available memory for each alive node in MB."""ray_nodes_info=get_ray_nodes_info()available_mems=[]fornodeid,infoinray_nodes_info.items():available_mems.append(info["free_memory"])returnavailable_mems
[docs]defray_available_gpu_memories():"""Available gpu memory of each gpu card for each alive node in MB."""ray_nodes_info=get_ray_nodes_info()available_gpu_mems=[]fornodeid,infoinray_nodes_info.items():available_gpu_mems.extend(info["free_gpus_memory"])returnavailable_gpu_mems