diff --git a/config/platform-infra/selfmedia-voice.yaml b/config/platform-infra/selfmedia-voice.yaml index 13cd4bb8..322204c0 100644 --- a/config/platform-infra/selfmedia-voice.yaml +++ b/config/platform-infra/selfmedia-voice.yaml @@ -128,10 +128,7 @@ runtime: max: 2.0 cudaAllocator: expandable_segments:True,max_split_size_mb:128 cudaModuleLoading: LAZY - onnxCuda: - gpuMemLimitMiB: 256 - arenaExtendStrategy: kSameAsRequested - cudnnConvUseMaxWorkspace: false + speechTokenizerProvider: CPUExecutionProvider operations: applyTimeoutSeconds: 3600 smoke: diff --git a/deploy/selfmedia-voice/api.py b/deploy/selfmedia-voice/api.py index 88ef4515..ae55a9e2 100644 --- a/deploy/selfmedia-voice/api.py +++ b/deploy/selfmedia-voice/api.py @@ -28,8 +28,7 @@ class Runtime: def __init__(self) -> None: self.model = None self.loaded_at = None - self.onnx_providers = None - self.onnx_provider_options = None + self.speech_tokenizer_providers = None self.error = None self.lock = threading.Lock() @@ -38,15 +37,19 @@ class Runtime: if not torch.cuda.is_available(): raise RuntimeError("CUDA is unavailable") device = os.environ["VOICE_DEVICE"] - if not device.startswith("cuda:"): - raise RuntimeError("VOICE_DEVICE must select a CUDA device") + if device != "cuda:0": + raise RuntimeError("VOICE_DEVICE must be cuda:0") + if not env_bool("VOICE_FP16"): + raise RuntimeError("VOICE_FP16 must remain enabled for core CUDA inference") torch.cuda.set_device(int(device.split(":", 1)[1])) original_inference_session = onnxruntime.InferenceSession - cuda_options = onnx_cuda_provider_options() + speech_tokenizer_provider = os.environ["VOICE_SPEECH_TOKENIZER_PROVIDER"] + if speech_tokenizer_provider != "CPUExecutionProvider": + raise RuntimeError("VOICE_SPEECH_TOKENIZER_PROVIDER must be CPUExecutionProvider") def inference_session(*args, **kwargs): if kwargs.get("providers") == ["CUDAExecutionProvider"]: - kwargs["providers"] = [("CUDAExecutionProvider", cuda_options)] + kwargs["providers"] = [speech_tokenizer_provider] return original_inference_session(*args, **kwargs) onnxruntime.InferenceSession = inference_session @@ -59,18 +62,9 @@ class Runtime: ) finally: onnxruntime.InferenceSession = original_inference_session - self.onnx_providers = self.model.frontend.speech_tokenizer_session.get_providers() - self.onnx_provider_options = self.model.frontend.speech_tokenizer_session.get_provider_options() - if not self.onnx_providers or self.onnx_providers[0] != "CUDAExecutionProvider": - raise RuntimeError(f"CUDAExecutionProvider was not instantiated: {self.onnx_providers}") - actual_cuda_options = self.onnx_provider_options.get("CUDAExecutionProvider", {}) - mismatches = { - key: {"expected": value, "actual": actual_cuda_options.get(key)} - for key, value in cuda_options.items() - if actual_cuda_options.get(key) != value - } - if mismatches: - raise RuntimeError(f"CUDAExecutionProvider options were not applied: {mismatches}") + self.speech_tokenizer_providers = self.model.frontend.speech_tokenizer_session.get_providers() + if self.speech_tokenizer_providers != [speech_tokenizer_provider]: + raise RuntimeError(f"speech tokenizer provider mismatch: {self.speech_tokenizer_providers}") self.loaded_at = time.time() except Exception as error: self.error = f"{type(error).__name__}: {error}" @@ -84,17 +78,6 @@ def env_bool(name: str) -> bool: return value == "true" -def onnx_cuda_provider_options() -> dict[str, str]: - gpu_mem_limit_mib = int(os.environ["VOICE_ONNX_CUDA_GPU_MEM_LIMIT_MIB"]) - if gpu_mem_limit_mib <= 0: - raise RuntimeError("VOICE_ONNX_CUDA_GPU_MEM_LIMIT_MIB must be positive") - return { - "gpu_mem_limit": str(gpu_mem_limit_mib * 1024 * 1024), - "arena_extend_strategy": os.environ["VOICE_ONNX_CUDA_ARENA_EXTEND_STRATEGY"], - "cudnn_conv_use_max_workspace": "1" if env_bool("VOICE_ONNX_CUDA_CUDNN_CONV_USE_MAX_WORKSPACE") else "0", - } - - runtime = Runtime() @@ -118,19 +101,13 @@ def health() -> dict: "torchaudio": torchaudio.__version__, "cuda": torch.version.cuda, "gpu": torch.cuda.get_device_name(torch.cuda.current_device()) if torch.cuda.is_available() else None, - "device": os.environ["VOICE_DEVICE"], - "fp16": env_bool("VOICE_FP16"), + "coreInferenceDevice": os.environ["VOICE_DEVICE"], + "coreInferenceFp16": env_bool("VOICE_FP16"), "loadTrt": env_bool("VOICE_LOAD_TRT"), "loadVllm": env_bool("VOICE_LOAD_VLLM"), "workers": int(os.environ["VOICE_WORKERS"]), "concurrency": os.environ["VOICE_CONCURRENCY"], - "onnxProviders": runtime.onnx_providers, - "onnxProviderOptions": runtime.onnx_provider_options, - "onnxCudaMemory": { - "gpuMemLimitMiB": int(os.environ["VOICE_ONNX_CUDA_GPU_MEM_LIMIT_MIB"]), - "arenaExtendStrategy": os.environ["VOICE_ONNX_CUDA_ARENA_EXTEND_STRATEGY"], - "cudnnConvUseMaxWorkspace": env_bool("VOICE_ONNX_CUDA_CUDNN_CONV_USE_MAX_WORKSPACE"), - }, + "speechTokenizerProviders": runtime.speech_tokenizer_providers, "loadedAt": runtime.loaded_at, "error": runtime.error, } diff --git a/deploy/selfmedia-voice/compose.d518.yaml b/deploy/selfmedia-voice/compose.d518.yaml index 4d3f8bac..d8a21f53 100644 --- a/deploy/selfmedia-voice/compose.d518.yaml +++ b/deploy/selfmedia-voice/compose.d518.yaml @@ -67,9 +67,7 @@ services: VOICE_PORT: ${VOICE_PORT} PYTORCH_CUDA_ALLOC_CONF: ${PYTORCH_CUDA_ALLOC_CONF} CUDA_MODULE_LOADING: ${CUDA_MODULE_LOADING} - VOICE_ONNX_CUDA_GPU_MEM_LIMIT_MIB: ${VOICE_ONNX_CUDA_GPU_MEM_LIMIT_MIB} - VOICE_ONNX_CUDA_ARENA_EXTEND_STRATEGY: ${VOICE_ONNX_CUDA_ARENA_EXTEND_STRATEGY} - VOICE_ONNX_CUDA_CUDNN_CONV_USE_MAX_WORKSPACE: ${VOICE_ONNX_CUDA_CUDNN_CONV_USE_MAX_WORKSPACE} + VOICE_SPEECH_TOKENIZER_PROVIDER: ${VOICE_SPEECH_TOKENIZER_PROVIDER} LD_LIBRARY_PATH: ${TORCH_LIBRARY_PATH} volumes: - ${VOICE_CACHE_DIR}/model:/model:ro diff --git a/scripts/src/platform-infra-selfmedia-voice.ts b/scripts/src/platform-infra-selfmedia-voice.ts index e77c1dd5..6998fc0e 100644 --- a/scripts/src/platform-infra-selfmedia-voice.ts +++ b/scripts/src/platform-infra-selfmedia-voice.ts @@ -68,7 +68,7 @@ interface VoiceConfig { speed: { min: number; max: number }; cudaAllocator: string; cudaModuleLoading: string; - onnxCuda: { gpuMemLimitMiB: number; arenaExtendStrategy: "kSameAsRequested" | "kNextPowerOfTwo"; cudnnConvUseMaxWorkspace: boolean }; + speechTokenizerProvider: "CPUExecutionProvider"; }; operations: { applyTimeoutSeconds: number }; smoke: { healthPath: string; speechPath: string; text: string; voice: string; timeoutSeconds: number }; @@ -208,10 +208,7 @@ function readConfig(): VoiceConfig { if (config.runtime.inference.concurrency !== "serial") throw new Error(`${configLabel}.runtime.inference.concurrency must be serial`); if (config.runtime.inference.referenceVoice !== config.image.referenceVoice.id) throw new Error(`${configLabel}.runtime.inference.referenceVoice must select image.referenceVoice.id`); if (typeof config.runtime.inference.speed.min !== "number" || typeof config.runtime.inference.speed.max !== "number" || config.runtime.inference.speed.min >= config.runtime.inference.speed.max) throw new Error(`${configLabel}.runtime.inference.speed must declare an increasing numeric range`); - requireInteger(config.runtime.inference.onnxCuda.gpuMemLimitMiB, "runtime.inference.onnxCuda.gpuMemLimitMiB"); - if (config.runtime.inference.onnxCuda.gpuMemLimitMiB <= 0) throw new Error(`${configLabel}.runtime.inference.onnxCuda.gpuMemLimitMiB must be positive`); - if (!["kSameAsRequested", "kNextPowerOfTwo"].includes(config.runtime.inference.onnxCuda.arenaExtendStrategy)) throw new Error(`${configLabel}.runtime.inference.onnxCuda.arenaExtendStrategy is unsupported`); - if (typeof config.runtime.inference.onnxCuda.cudnnConvUseMaxWorkspace !== "boolean") throw new Error(`${configLabel}.runtime.inference.onnxCuda.cudnnConvUseMaxWorkspace must be a boolean`); + if (config.runtime.inference.speechTokenizerProvider !== "CPUExecutionProvider") throw new Error(`${configLabel}.runtime.inference.speechTokenizerProvider must be CPUExecutionProvider`); return config; } @@ -388,9 +385,7 @@ function runtimeEnv(config: VoiceConfig, target: VoiceTarget): string { VOICE_SPEED_MAX: config.runtime.inference.speed.max, PYTORCH_CUDA_ALLOC_CONF: config.runtime.inference.cudaAllocator, CUDA_MODULE_LOADING: config.runtime.inference.cudaModuleLoading, - VOICE_ONNX_CUDA_GPU_MEM_LIMIT_MIB: config.runtime.inference.onnxCuda.gpuMemLimitMiB, - VOICE_ONNX_CUDA_ARENA_EXTEND_STRATEGY: config.runtime.inference.onnxCuda.arenaExtendStrategy, - VOICE_ONNX_CUDA_CUDNN_CONV_USE_MAX_WORKSPACE: String(config.runtime.inference.onnxCuda.cudnnConvUseMaxWorkspace), + VOICE_SPEECH_TOKENIZER_PROVIDER: config.runtime.inference.speechTokenizerProvider, FRPC_IMAGE: config.frp.images.frpc, }; return Object.entries(values).map(([key, value]) => `${key}=${value}`).join("\n") + "\n";