stash: refactoring using yoke grind
This commit is contained in:
parent
73cfaad5d6
commit
599e8e870b
10 changed files with 1566 additions and 1715 deletions
|
|
@ -49,16 +49,6 @@ pub(super) struct NodeImageRequest {
|
||||||
pub(super) enabled: bool,
|
pub(super) enabled: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
pub(super) struct PreparedNodeImage {
|
|
||||||
pub(super) image_ref: String,
|
|
||||||
pub(super) tag: String,
|
|
||||||
pub(super) already_available: bool,
|
|
||||||
pub(super) built: bool,
|
|
||||||
pub(super) pushed: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub(super) enum NodeImageProgressEventKind {
|
pub(super) enum NodeImageProgressEventKind {
|
||||||
ImageReference {
|
ImageReference {
|
||||||
|
|
@ -140,7 +130,7 @@ struct RealImageCommandRunner;
|
||||||
pub(super) fn prepare_node_image_with_progress(
|
pub(super) fn prepare_node_image_with_progress(
|
||||||
request: NodeImageRequest,
|
request: NodeImageRequest,
|
||||||
progress: Option<&mut dyn NodeImageProgressSink>,
|
progress: Option<&mut dyn NodeImageProgressSink>,
|
||||||
) -> Result<PreparedNodeImage, String> {
|
) -> Result<String, String> {
|
||||||
let mut progress = progress;
|
let mut progress = progress;
|
||||||
let mut runner = RealImageCommandRunner;
|
let mut runner = RealImageCommandRunner;
|
||||||
prepare_node_image_inner(request, &mut progress, &mut runner)
|
prepare_node_image_inner(request, &mut progress, &mut runner)
|
||||||
|
|
@ -150,16 +140,10 @@ fn prepare_node_image_inner(
|
||||||
request: NodeImageRequest,
|
request: NodeImageRequest,
|
||||||
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
||||||
runner: &mut dyn ImageCommandRunner,
|
runner: &mut dyn ImageCommandRunner,
|
||||||
) -> Result<PreparedNodeImage, String> {
|
) -> Result<String, String> {
|
||||||
emit_image_reference(progress, "requested", &request.requested_image);
|
emit_image_reference(progress, "requested", &request.requested_image);
|
||||||
if !request.enabled {
|
if !request.enabled {
|
||||||
return Ok(PreparedNodeImage {
|
return Ok(request.requested_image);
|
||||||
image_ref: request.requested_image,
|
|
||||||
tag: String::new(),
|
|
||||||
already_available: false,
|
|
||||||
built: false,
|
|
||||||
pushed: false,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
emit_image_reference(progress, "base", &request.base_image);
|
emit_image_reference(progress, "base", &request.base_image);
|
||||||
|
|
@ -208,16 +192,9 @@ fn prepare_node_image_inner(
|
||||||
docker_image_labels_match(runner, &root, &image_ref, &expected_node_labels)?;
|
docker_image_labels_match(runner, &root, &image_ref, &expected_node_labels)?;
|
||||||
let remote_available = remote_required && runner.docker_manifest_exists(&root, &image_ref);
|
let remote_available = remote_required && runner.docker_manifest_exists(&root, &image_ref);
|
||||||
if !request.force_refresh && remote_required && remote_available {
|
if !request.force_refresh && remote_required && remote_available {
|
||||||
let pushed =
|
|
||||||
ensure_aliases_for_remote(runner, progress, &root, &image_ref, &image, &alias_tags)?;
|
ensure_aliases_for_remote(runner, progress, &root, &image_ref, &image, &alias_tags)?;
|
||||||
prune_old_dirty_images(runner, &root, &image, &tag);
|
prune_old_dirty_images(runner, &root, &image, &tag);
|
||||||
return Ok(PreparedNodeImage {
|
return Ok(image_ref);
|
||||||
image_ref,
|
|
||||||
tag,
|
|
||||||
already_available: true,
|
|
||||||
built: false,
|
|
||||||
pushed,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
if !request.force_refresh && remote_required && local_image_matches {
|
if !request.force_refresh && remote_required && local_image_matches {
|
||||||
ensure_aliases_local(runner, progress, &root, &image_ref, &image, &alias_tags)?;
|
ensure_aliases_local(runner, progress, &root, &image_ref, &image, &alias_tags)?;
|
||||||
|
|
@ -226,24 +203,12 @@ fn prepare_node_image_inner(
|
||||||
push_image(runner, progress, &root, &alias)?;
|
push_image(runner, progress, &root, &alias)?;
|
||||||
}
|
}
|
||||||
prune_old_dirty_images(runner, &root, &image, &tag);
|
prune_old_dirty_images(runner, &root, &image, &tag);
|
||||||
return Ok(PreparedNodeImage {
|
return Ok(image_ref);
|
||||||
image_ref,
|
|
||||||
tag,
|
|
||||||
already_available: true,
|
|
||||||
built: false,
|
|
||||||
pushed: true,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
if !request.force_refresh && !remote_required && local_image_matches {
|
if !request.force_refresh && !remote_required && local_image_matches {
|
||||||
ensure_aliases_local(runner, progress, &root, &image_ref, &image, &alias_tags)?;
|
ensure_aliases_local(runner, progress, &root, &image_ref, &image, &alias_tags)?;
|
||||||
prune_old_dirty_images(runner, &root, &image, &tag);
|
prune_old_dirty_images(runner, &root, &image, &tag);
|
||||||
return Ok(PreparedNodeImage {
|
return Ok(image_ref);
|
||||||
image_ref,
|
|
||||||
tag,
|
|
||||||
already_available: true,
|
|
||||||
built: false,
|
|
||||||
pushed: false,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
let base_image_matches =
|
let base_image_matches =
|
||||||
docker_image_labels_match(runner, &root, &request.base_image, &expected_base_labels)?;
|
docker_image_labels_match(runner, &root, &request.base_image, &expected_base_labels)?;
|
||||||
|
|
@ -294,23 +259,15 @@ fn prepare_node_image_inner(
|
||||||
)?;
|
)?;
|
||||||
ensure_aliases_local(runner, progress, &root, &image_ref, &image, &alias_tags)?;
|
ensure_aliases_local(runner, progress, &root, &image_ref, &image, &alias_tags)?;
|
||||||
|
|
||||||
let mut pushed = false;
|
|
||||||
if remote_required {
|
if remote_required {
|
||||||
push_image(runner, progress, &root, &image_ref)?;
|
push_image(runner, progress, &root, &image_ref)?;
|
||||||
pushed = true;
|
|
||||||
for alias in alias_refs(&image, &alias_tags) {
|
for alias in alias_refs(&image, &alias_tags) {
|
||||||
push_image(runner, progress, &root, &alias)?;
|
push_image(runner, progress, &root, &alias)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
prune_old_dirty_images(runner, &root, &image, &tag);
|
prune_old_dirty_images(runner, &root, &image, &tag);
|
||||||
Ok(PreparedNodeImage {
|
Ok(image_ref)
|
||||||
image_ref,
|
|
||||||
tag,
|
|
||||||
already_available: false,
|
|
||||||
built: true,
|
|
||||||
pushed,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn workspace_root() -> Result<PathBuf, String> {
|
fn workspace_root() -> Result<PathBuf, String> {
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@ use signal_hook::iterator::Signals;
|
||||||
use crate::chat::config as chat_config;
|
use crate::chat::config as chat_config;
|
||||||
use crate::chat::node_image::{
|
use crate::chat::node_image::{
|
||||||
NodeImageProgressEvent, NodeImageProgressEventKind, NodeImageProgressSink, NodeImageProvider,
|
NodeImageProgressEvent, NodeImageProgressEventKind, NodeImageProgressSink, NodeImageProvider,
|
||||||
NodeImageRequest, PreparedNodeImage, prepare_node_image_with_progress,
|
NodeImageRequest, prepare_node_image_with_progress,
|
||||||
};
|
};
|
||||||
use crate::node_provisioning::{ProviderKind, provider_kind};
|
use crate::node_provisioning::{ProviderKind, provider_kind};
|
||||||
use crate::observability::{benchmark, frame_archive::FrameArchive};
|
use crate::observability::{benchmark, frame_archive::FrameArchive};
|
||||||
|
|
@ -775,73 +775,21 @@ impl Config {
|
||||||
let args = ParsedArgs::parse(provided_args)?;
|
let args = ParsedArgs::parse(provided_args)?;
|
||||||
let loaded = load_chat_config(args.config_path.as_deref())?;
|
let loaded = load_chat_config(args.config_path.as_deref())?;
|
||||||
let toml = loaded.overlay;
|
let toml = loaded.overlay;
|
||||||
let provider = provider_from_sources(args.provider, toml.provider.kind.as_deref())?;
|
let provider = provider_from_sources(args.provider.clone(), toml.provider.kind.as_deref())?;
|
||||||
let node_image = first_non_empty([toml.image.node.clone()]).unwrap_or_default();
|
let node_image = first_non_empty([toml.image.node.clone()]).unwrap_or_default();
|
||||||
if provider != provider_kind::process() && node_image.is_empty() {
|
if provider != provider_kind::process() && node_image.is_empty() {
|
||||||
return Err("node image is required for docker or vastai provider".to_owned());
|
return Err("node image is required for docker or vastai provider".to_owned());
|
||||||
}
|
}
|
||||||
let pipeline_stages = args
|
let pipeline_stages = Self::pipeline_stages(&args, &toml)?;
|
||||||
.pipeline_stages
|
let max_tokens = Self::max_tokens(&toml)?;
|
||||||
.or(toml.runtime.pipeline_stages)
|
|
||||||
.unwrap_or(1);
|
|
||||||
if pipeline_stages == 0 {
|
|
||||||
return Err("--pipeline-stages must be greater than 0".to_owned());
|
|
||||||
}
|
|
||||||
let max_tokens = toml.runtime.max_tokens.unwrap_or(DEFAULT_MAX_TOKENS);
|
|
||||||
if max_tokens == 0 {
|
|
||||||
return Err("[runtime].max_tokens must be greater than 0".to_owned());
|
|
||||||
}
|
|
||||||
let gpu_run = args.gpu || env_flag(MVP_CHAT_GPU_RUN_ENV, false);
|
let gpu_run = args.gpu || env_flag(MVP_CHAT_GPU_RUN_ENV, false);
|
||||||
let endpoint_addr_mask = match first_non_empty([
|
let endpoint_addr_mask = Self::endpoint_addr_mask(&args, &toml)?;
|
||||||
args.endpoint_addr_mask.clone(),
|
let (relay_mode, relay_url) = Self::relay_settings(&args, &toml, endpoint_addr_mask)?;
|
||||||
toml.relay.endpoint_addr_mask.clone(),
|
let cached_model = Self::cached_model_source(&args, gpu_run, &provider)
|
||||||
]) {
|
|
||||||
Some(mask) => EndpointAddrMask::parse(&mask)?,
|
|
||||||
None => EndpointAddrMask::Full,
|
|
||||||
};
|
|
||||||
let relay_mode = first_non_empty([args.relay_mode.clone(), toml.relay.mode.clone()]);
|
|
||||||
let mut relay_url = first_non_empty([args.relay_url.clone(), toml.relay.url.clone()]);
|
|
||||||
if endpoint_addr_mask.requires_relay() && relay_url.is_none() {
|
|
||||||
relay_url = first_non_empty([toml.vastai.relay_url.clone()]);
|
|
||||||
}
|
|
||||||
if endpoint_addr_mask.requires_relay() && relay_url.is_none() {
|
|
||||||
return Err("relay-only endpoint address mask requires [relay].url, --relay-url, or [vastai].relay_url".to_owned());
|
|
||||||
}
|
|
||||||
let relay_mode = relay_mode.or_else(|| relay_url.as_ref().map(|_| "default".to_owned()));
|
|
||||||
let cached_model_source = match args.cached_model {
|
|
||||||
Some(source) => Some(source),
|
|
||||||
None if gpu_run && provider == provider_kind::process() => {
|
|
||||||
Some(CachedModelSource::Discover)
|
|
||||||
}
|
|
||||||
None => None,
|
|
||||||
};
|
|
||||||
let cached_model = cached_model_source
|
|
||||||
.map(CachedModelConfig::from_source)
|
.map(CachedModelConfig::from_source)
|
||||||
.transpose()?;
|
.transpose()?;
|
||||||
let model = if provider == provider_kind::vastai() {
|
let model = Self::model_config(&provider, &toml, cached_model.as_ref())?;
|
||||||
match &cached_model {
|
let datastream_frame_log = Self::datastream_frame_log(&args, &toml);
|
||||||
Some(cached_model) => {
|
|
||||||
vastai_model_config_for_cached_model(toml.model.clone(), cached_model)?
|
|
||||||
}
|
|
||||||
None => toml.model.clone(),
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
toml.model.clone()
|
|
||||||
};
|
|
||||||
let datastream_frame_log = if args.dump_logs {
|
|
||||||
Some(
|
|
||||||
args.dump_log_path
|
|
||||||
.unwrap_or_else(|| PathBuf::from("mvp-chat.log")),
|
|
||||||
)
|
|
||||||
} else if toml.observability.dump_logs.unwrap_or(false) {
|
|
||||||
Some(
|
|
||||||
first_non_empty([toml.observability.dump_log_path.clone()])
|
|
||||||
.map(PathBuf::from)
|
|
||||||
.unwrap_or_else(|| PathBuf::from("mvp-chat.log")),
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
let vastai = if provider == provider_kind::vastai() {
|
let vastai = if provider == provider_kind::vastai() {
|
||||||
Some(resolve_vastai_config(&toml.vastai, &node_image)?)
|
Some(resolve_vastai_config(&toml.vastai, &node_image)?)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -871,9 +819,114 @@ impl Config {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn pipeline_stages(args: &ParsedArgs, toml: &ChatTomlConfig) -> Result<u32, String> {
|
||||||
|
let pipeline_stages = args
|
||||||
|
.pipeline_stages
|
||||||
|
.or(toml.runtime.pipeline_stages)
|
||||||
|
.unwrap_or(1);
|
||||||
|
if pipeline_stages == 0 {
|
||||||
|
return Err("--pipeline-stages must be greater than 0".to_owned());
|
||||||
|
}
|
||||||
|
Ok(pipeline_stages)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn max_tokens(toml: &ChatTomlConfig) -> Result<u32, String> {
|
||||||
|
let max_tokens = toml.runtime.max_tokens.unwrap_or(DEFAULT_MAX_TOKENS);
|
||||||
|
if max_tokens == 0 {
|
||||||
|
return Err("[runtime].max_tokens must be greater than 0".to_owned());
|
||||||
|
}
|
||||||
|
Ok(max_tokens)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn endpoint_addr_mask(
|
||||||
|
args: &ParsedArgs,
|
||||||
|
toml: &ChatTomlConfig,
|
||||||
|
) -> Result<EndpointAddrMask, String> {
|
||||||
|
match first_non_empty([
|
||||||
|
args.endpoint_addr_mask.clone(),
|
||||||
|
toml.relay.endpoint_addr_mask.clone(),
|
||||||
|
]) {
|
||||||
|
Some(mask) => EndpointAddrMask::parse(&mask),
|
||||||
|
None => Ok(EndpointAddrMask::Full),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn relay_settings(
|
||||||
|
args: &ParsedArgs,
|
||||||
|
toml: &ChatTomlConfig,
|
||||||
|
endpoint_addr_mask: EndpointAddrMask,
|
||||||
|
) -> Result<(Option<String>, Option<String>), String> {
|
||||||
|
let relay_mode = first_non_empty([args.relay_mode.clone(), toml.relay.mode.clone()]);
|
||||||
|
let mut relay_url = first_non_empty([args.relay_url.clone(), toml.relay.url.clone()]);
|
||||||
|
if endpoint_addr_mask.requires_relay() && relay_url.is_none() {
|
||||||
|
relay_url = first_non_empty([toml.vastai.relay_url.clone()]);
|
||||||
|
}
|
||||||
|
if endpoint_addr_mask.requires_relay() && relay_url.is_none() {
|
||||||
|
return Err("relay-only endpoint address mask requires [relay].url, --relay-url, or [vastai].relay_url".to_owned());
|
||||||
|
}
|
||||||
|
let relay_mode = relay_mode.or_else(|| relay_url.as_ref().map(|_| "default".to_owned()));
|
||||||
|
Ok((relay_mode, relay_url))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cached_model_source(
|
||||||
|
args: &ParsedArgs,
|
||||||
|
gpu_run: bool,
|
||||||
|
provider: &ProviderKind,
|
||||||
|
) -> Option<CachedModelSource> {
|
||||||
|
match &args.cached_model {
|
||||||
|
Some(source) => Some(source.clone()),
|
||||||
|
None if gpu_run && provider == &provider_kind::process() => {
|
||||||
|
Some(CachedModelSource::Discover)
|
||||||
|
}
|
||||||
|
None => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn model_config(
|
||||||
|
provider: &ProviderKind,
|
||||||
|
toml: &ChatTomlConfig,
|
||||||
|
cached_model: Option<&CachedModelConfig>,
|
||||||
|
) -> Result<ChatModelConfig, String> {
|
||||||
|
if provider != &provider_kind::vastai() {
|
||||||
|
return Ok(toml.model.clone());
|
||||||
|
}
|
||||||
|
match cached_model {
|
||||||
|
Some(cached_model) => {
|
||||||
|
vastai_model_config_for_cached_model(toml.model.clone(), cached_model)
|
||||||
|
}
|
||||||
|
None => Ok(toml.model.clone()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn datastream_frame_log(args: &ParsedArgs, toml: &ChatTomlConfig) -> Option<PathBuf> {
|
||||||
|
if args.dump_logs {
|
||||||
|
return Some(
|
||||||
|
args.dump_log_path
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| PathBuf::from("mvp-chat.log")),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if !toml.observability.dump_logs.unwrap_or(false) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(
|
||||||
|
first_non_empty([toml.observability.dump_log_path.clone()])
|
||||||
|
.map(PathBuf::from)
|
||||||
|
.unwrap_or_else(|| PathBuf::from("mvp-chat.log")),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// The orchestrator launch spec is still pending. These flags are the current adapter;
|
// The orchestrator launch spec is still pending. These flags are the current adapter;
|
||||||
// adjust this mapping when the approved orchestrator launch contract is finalized.
|
// adjust this mapping when the approved orchestrator launch contract is finalized.
|
||||||
fn orchestrator_cli_args(&self, image_ref: &str) -> Vec<String> {
|
fn orchestrator_cli_args(&self, image_ref: &str) -> Vec<String> {
|
||||||
|
macro_rules! push_opt {
|
||||||
|
($args:ident, $option:expr, $flag:expr, |$value:ident| $arg:expr) => {
|
||||||
|
if let Some($value) = $option {
|
||||||
|
$args.extend([$flag.to_owned(), $arg]);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
let mut args = vec![
|
let mut args = vec![
|
||||||
"--provider".to_owned(),
|
"--provider".to_owned(),
|
||||||
self.provider.as_str().to_owned(),
|
self.provider.as_str().to_owned(),
|
||||||
|
|
@ -889,27 +942,36 @@ impl Config {
|
||||||
self.pipeline_stages.to_string(),
|
self.pipeline_stages.to_string(),
|
||||||
"--dashboard".to_owned(),
|
"--dashboard".to_owned(),
|
||||||
];
|
];
|
||||||
if let Some(model_id) = &self.model.id {
|
push_opt!(args, &self.model.id, "--model-id", |model_id| model_id
|
||||||
args.extend(["--model-id".to_owned(), model_id.clone()]);
|
.clone());
|
||||||
}
|
push_opt!(
|
||||||
if let Some(path) = &self.model.gguf_local_path {
|
args,
|
||||||
args.extend(["--gguf-local-path".to_owned(), path.clone()]);
|
&self.model.gguf_local_path,
|
||||||
}
|
"--gguf-local-path",
|
||||||
if let Some(repo) = &self.model.gguf_repo {
|
|path| path.clone()
|
||||||
args.extend(["--gguf-repo".to_owned(), repo.clone()]);
|
);
|
||||||
}
|
push_opt!(args, &self.model.gguf_repo, "--gguf-repo", |repo| repo
|
||||||
if let Some(file) = &self.model.gguf_file {
|
.clone());
|
||||||
args.extend(["--gguf-file".to_owned(), file.clone()]);
|
push_opt!(args, &self.model.gguf_file, "--gguf-file", |file| file
|
||||||
}
|
.clone());
|
||||||
if let Some(revision) = &self.model.gguf_revision {
|
push_opt!(
|
||||||
args.extend(["--gguf-revision".to_owned(), revision.clone()]);
|
args,
|
||||||
}
|
&self.model.gguf_revision,
|
||||||
if let Some(path) = &self.model.tokenizer_local_path {
|
"--gguf-revision",
|
||||||
args.extend(["--tokenizer-local-path".to_owned(), path.clone()]);
|
|revision| { revision.clone() }
|
||||||
}
|
);
|
||||||
if let Some(max_context) = self.model.max_context {
|
push_opt!(
|
||||||
args.extend(["--max-context".to_owned(), max_context.to_string()]);
|
args,
|
||||||
}
|
&self.model.tokenizer_local_path,
|
||||||
|
"--tokenizer-local-path",
|
||||||
|
|path| path.clone()
|
||||||
|
);
|
||||||
|
push_opt!(
|
||||||
|
args,
|
||||||
|
self.model.max_context,
|
||||||
|
"--max-context",
|
||||||
|
|max_context| { max_context.to_string() }
|
||||||
|
);
|
||||||
if self.provider == provider_kind::process() {
|
if self.provider == provider_kind::process() {
|
||||||
args.extend([
|
args.extend([
|
||||||
"--worker-bin".to_owned(),
|
"--worker-bin".to_owned(),
|
||||||
|
|
@ -922,18 +984,14 @@ impl Config {
|
||||||
cached_model.host_path.to_string_lossy().to_string(),
|
cached_model.host_path.to_string_lossy().to_string(),
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
if let Some(path) = &self.datastream_frame_log {
|
push_opt!(
|
||||||
args.extend([
|
args,
|
||||||
"--datastream-frame-log".to_owned(),
|
&self.datastream_frame_log,
|
||||||
path.to_string_lossy().to_string(),
|
"--datastream-frame-log",
|
||||||
]);
|
|path| { path.to_string_lossy().to_string() }
|
||||||
}
|
);
|
||||||
if let Some(mode) = &self.relay_mode {
|
push_opt!(args, &self.relay_mode, "--relay-mode", |mode| mode.clone());
|
||||||
args.extend(["--relay-mode".to_owned(), mode.clone()]);
|
push_opt!(args, &self.relay_url, "--relay-url", |url| url.clone());
|
||||||
}
|
|
||||||
if let Some(url) = &self.relay_url {
|
|
||||||
args.extend(["--relay-url".to_owned(), url.clone()]);
|
|
||||||
}
|
|
||||||
if self.endpoint_addr_mask != EndpointAddrMask::Full {
|
if self.endpoint_addr_mask != EndpointAddrMask::Full {
|
||||||
args.extend([
|
args.extend([
|
||||||
"--endpoint-addr-mask".to_owned(),
|
"--endpoint-addr-mask".to_owned(),
|
||||||
|
|
@ -946,39 +1004,41 @@ impl Config {
|
||||||
vastai.bootstrap_command.clone(),
|
vastai.bootstrap_command.clone(),
|
||||||
"--no-vastai-confirm-lease".to_owned(),
|
"--no-vastai-confirm-lease".to_owned(),
|
||||||
]);
|
]);
|
||||||
if let Some(disk_gb) = vastai.disk_gb {
|
push_opt!(args, vastai.disk_gb, "--vastai-disk-gb", |disk_gb| disk_gb
|
||||||
args.extend(["--vastai-disk-gb".to_owned(), disk_gb.to_string()]);
|
.to_string());
|
||||||
}
|
push_opt!(args, &vastai.gpu_name, "--vastai-gpu-name", |gpu_name| {
|
||||||
if let Some(gpu_name) = &vastai.gpu_name {
|
gpu_name.clone()
|
||||||
args.extend(["--vastai-gpu-name".to_owned(), gpu_name.clone()]);
|
});
|
||||||
}
|
push_opt!(
|
||||||
if let Some(min_gpu_ram_mb) = vastai.min_gpu_ram_mb {
|
args,
|
||||||
args.extend([
|
vastai.min_gpu_ram_mb,
|
||||||
"--vastai-min-gpu-ram-mb".to_owned(),
|
"--vastai-min-gpu-ram-mb",
|
||||||
min_gpu_ram_mb.to_string(),
|
|min_gpu_ram_mb| min_gpu_ram_mb.to_string()
|
||||||
]);
|
);
|
||||||
}
|
push_opt!(
|
||||||
if let Some(min_down_mbps) = vastai.min_down_mbps {
|
args,
|
||||||
args.extend([
|
vastai.min_down_mbps,
|
||||||
"--vastai-min-down-mbps".to_owned(),
|
"--vastai-min-down-mbps",
|
||||||
min_down_mbps.to_string(),
|
|min_down_mbps| min_down_mbps.to_string()
|
||||||
]);
|
);
|
||||||
}
|
push_opt!(
|
||||||
if let Some(min_up_mbps) = vastai.min_up_mbps {
|
args,
|
||||||
args.extend(["--vastai-min-up-mbps".to_owned(), min_up_mbps.to_string()]);
|
vastai.min_up_mbps,
|
||||||
}
|
"--vastai-min-up-mbps",
|
||||||
if let Some(max_dph_total) = vastai.max_dph_total {
|
|min_up_mbps| { min_up_mbps.to_string() }
|
||||||
args.extend([
|
);
|
||||||
"--vastai-max-dph-total".to_owned(),
|
push_opt!(
|
||||||
max_dph_total.to_string(),
|
args,
|
||||||
]);
|
vastai.max_dph_total,
|
||||||
}
|
"--vastai-max-dph-total",
|
||||||
if let Some(min_reliability) = vastai.min_reliability {
|
|max_dph_total| { max_dph_total.to_string() }
|
||||||
args.extend([
|
);
|
||||||
"--vastai-min-reliability".to_owned(),
|
push_opt!(
|
||||||
min_reliability.to_string(),
|
args,
|
||||||
]);
|
vastai.min_reliability,
|
||||||
}
|
"--vastai-min-reliability",
|
||||||
|
|min_reliability| min_reliability.to_string()
|
||||||
|
);
|
||||||
if let Some(require_verified) = vastai.require_verified {
|
if let Some(require_verified) = vastai.require_verified {
|
||||||
args.push(if require_verified {
|
args.push(if require_verified {
|
||||||
"--vastai-require-verified".to_owned()
|
"--vastai-require-verified".to_owned()
|
||||||
|
|
@ -989,12 +1049,14 @@ impl Config {
|
||||||
for host_id in &vastai.blacklist_hosts {
|
for host_id in &vastai.blacklist_hosts {
|
||||||
args.extend(["--vastai-blacklist-host".to_owned(), host_id.to_string()]);
|
args.extend(["--vastai-blacklist-host".to_owned(), host_id.to_string()]);
|
||||||
}
|
}
|
||||||
if let Some(onstart) = &vastai.onstart {
|
push_opt!(args, &vastai.onstart, "--vastai-onstart", |onstart| onstart
|
||||||
args.extend(["--vastai-onstart".to_owned(), onstart.clone()]);
|
.clone());
|
||||||
}
|
push_opt!(
|
||||||
if let Some(ssh_identity) = &vastai.ssh_identity {
|
args,
|
||||||
args.extend(["--vastai-ssh-identity".to_owned(), ssh_identity.clone()]);
|
&vastai.ssh_identity,
|
||||||
}
|
"--vastai-ssh-identity",
|
||||||
|
|ssh_identity| { ssh_identity.clone() }
|
||||||
|
);
|
||||||
}
|
}
|
||||||
args
|
args
|
||||||
}
|
}
|
||||||
|
|
@ -1044,6 +1106,70 @@ impl ParsedArgs {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn apply_provider_arg(&mut self, arg: &str) -> Result<bool, String> {
|
||||||
|
match arg {
|
||||||
|
"--help" | "-h" | "help" => self.help = true,
|
||||||
|
"--gpu" => self.gpu = true,
|
||||||
|
"--vastai" => self.set_provider_selector(provider_kind::vastai())?,
|
||||||
|
"--process" => self.set_provider_selector(provider_kind::process())?,
|
||||||
|
"--docker" => self.set_provider_selector(provider_kind::docker())?,
|
||||||
|
"--yes" | "-y" => self.vastai_yes = true,
|
||||||
|
"--dump-logs" => self.dump_logs = true,
|
||||||
|
"--cached-model" => self.cached_model = Some(CachedModelSource::Discover),
|
||||||
|
"--skip-rebuild" => self.skip_rebuild = true,
|
||||||
|
_ => return Ok(false),
|
||||||
|
}
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_config_arg<I>(&mut self, arg: &str, args: &mut I) -> Result<bool, String>
|
||||||
|
where
|
||||||
|
I: Iterator<Item = String>,
|
||||||
|
{
|
||||||
|
match arg {
|
||||||
|
"--config" => self.config_path = Some(PathBuf::from(next_arg(args, "--config")?)),
|
||||||
|
"--pipeline-stages" | "--pipeline-parallel" => {
|
||||||
|
if self.pipeline_stages.is_some() {
|
||||||
|
return Err("pipeline stage count was provided more than once".to_owned());
|
||||||
|
}
|
||||||
|
self.pipeline_stages = Some(parse_pipeline_stages_value(args, arg)?);
|
||||||
|
}
|
||||||
|
"--relay-mode" => self.relay_mode = Some(next_arg(args, "--relay-mode")?),
|
||||||
|
"--relay-url" => self.relay_url = Some(next_arg(args, "--relay-url")?),
|
||||||
|
"--endpoint-addr-mask" => {
|
||||||
|
self.endpoint_addr_mask = Some(next_arg(args, "--endpoint-addr-mask")?)
|
||||||
|
}
|
||||||
|
"--run-id" => {
|
||||||
|
let run_id: u64 = parse_next(args, "--run-id")?;
|
||||||
|
if run_id == 0 {
|
||||||
|
return Err("--run-id must be greater than 0".to_owned());
|
||||||
|
}
|
||||||
|
self.run_id = Some(run_id);
|
||||||
|
}
|
||||||
|
_ => return Ok(false),
|
||||||
|
}
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_assignment_arg(&mut self, arg: &str) -> Result<bool, String> {
|
||||||
|
if let Some(path) = arg.strip_prefix("--dump-logs=") {
|
||||||
|
if path.is_empty() {
|
||||||
|
return Err("--dump-logs path must not be empty".to_owned());
|
||||||
|
}
|
||||||
|
self.dump_logs = true;
|
||||||
|
self.dump_log_path = Some(PathBuf::from(path));
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
if let Some(path) = arg.strip_prefix("--cached-model=") {
|
||||||
|
if path.is_empty() {
|
||||||
|
return Err("--cached-model path must not be empty".to_owned());
|
||||||
|
}
|
||||||
|
self.cached_model = Some(CachedModelSource::Path(PathBuf::from(path)));
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
|
||||||
fn parse<I>(provided_args: I) -> Result<Self, String>
|
fn parse<I>(provided_args: I) -> Result<Self, String>
|
||||||
where
|
where
|
||||||
I: IntoIterator<Item = String>,
|
I: IntoIterator<Item = String>,
|
||||||
|
|
@ -1051,61 +1177,13 @@ impl ParsedArgs {
|
||||||
let mut parsed = Self::default();
|
let mut parsed = Self::default();
|
||||||
let mut args = provided_args.into_iter().peekable();
|
let mut args = provided_args.into_iter().peekable();
|
||||||
while let Some(arg) = args.next() {
|
while let Some(arg) = args.next() {
|
||||||
match arg.as_str() {
|
if parsed.apply_provider_arg(&arg)?
|
||||||
"--help" | "-h" | "help" => parsed.help = true,
|
|| parsed.apply_config_arg(&arg, &mut args)?
|
||||||
"--gpu" => parsed.gpu = true,
|
|| parsed.apply_assignment_arg(&arg)?
|
||||||
"--vastai" => parsed.set_provider_selector(provider_kind::vastai())?,
|
{
|
||||||
"--process" => parsed.set_provider_selector(provider_kind::process())?,
|
continue;
|
||||||
"--docker" => parsed.set_provider_selector(provider_kind::docker())?,
|
|
||||||
"--yes" | "-y" => parsed.vastai_yes = true,
|
|
||||||
"--config" => {
|
|
||||||
parsed.config_path = Some(PathBuf::from(next_arg(&mut args, "--config")?))
|
|
||||||
}
|
|
||||||
"--pipeline-stages" | "--pipeline-parallel" => {
|
|
||||||
if parsed.pipeline_stages.is_some() {
|
|
||||||
return Err("pipeline stage count was provided more than once".to_owned());
|
|
||||||
}
|
|
||||||
parsed.pipeline_stages =
|
|
||||||
Some(parse_pipeline_stages_value(&mut args, arg.as_str())?)
|
|
||||||
}
|
|
||||||
"--relay-mode" => parsed.relay_mode = Some(next_arg(&mut args, "--relay-mode")?),
|
|
||||||
"--relay-url" => parsed.relay_url = Some(next_arg(&mut args, "--relay-url")?),
|
|
||||||
"--endpoint-addr-mask" => {
|
|
||||||
parsed.endpoint_addr_mask = Some(next_arg(&mut args, "--endpoint-addr-mask")?)
|
|
||||||
}
|
|
||||||
"--run-id" => {
|
|
||||||
let run_id: u64 = parse_next(&mut args, "--run-id")?;
|
|
||||||
if run_id == 0 {
|
|
||||||
return Err("--run-id must be greater than 0".to_owned());
|
|
||||||
}
|
|
||||||
parsed.run_id = Some(run_id);
|
|
||||||
}
|
|
||||||
"--dump-logs" => {
|
|
||||||
parsed.dump_logs = true;
|
|
||||||
}
|
|
||||||
value if value.starts_with("--dump-logs=") => {
|
|
||||||
let path = value.strip_prefix("--dump-logs=").expect("prefix checked");
|
|
||||||
if path.is_empty() {
|
|
||||||
return Err("--dump-logs path must not be empty".to_owned());
|
|
||||||
}
|
|
||||||
parsed.dump_logs = true;
|
|
||||||
parsed.dump_log_path = Some(PathBuf::from(path));
|
|
||||||
}
|
|
||||||
"--cached-model" => {
|
|
||||||
parsed.cached_model = Some(CachedModelSource::Discover);
|
|
||||||
}
|
|
||||||
value if value.starts_with("--cached-model=") => {
|
|
||||||
let path = value
|
|
||||||
.strip_prefix("--cached-model=")
|
|
||||||
.expect("prefix checked");
|
|
||||||
if path.is_empty() {
|
|
||||||
return Err("--cached-model path must not be empty".to_owned());
|
|
||||||
}
|
|
||||||
parsed.cached_model = Some(CachedModelSource::Path(PathBuf::from(path)));
|
|
||||||
}
|
|
||||||
"--skip-rebuild" => parsed.skip_rebuild = true,
|
|
||||||
other => return Err(format!("unsupported mvp-chat argument {other:?}")),
|
|
||||||
}
|
}
|
||||||
|
return Err(format!("unsupported mvp-chat argument {arg:?}"));
|
||||||
}
|
}
|
||||||
Ok(parsed)
|
Ok(parsed)
|
||||||
}
|
}
|
||||||
|
|
@ -1511,7 +1589,7 @@ fn signal_orch_process_group(child: &Child, signal: libc::c_int) -> io::Result<(
|
||||||
fn prepare_node_image_progress_adapter(
|
fn prepare_node_image_progress_adapter(
|
||||||
request: NodeImageRequest,
|
request: NodeImageRequest,
|
||||||
progress: Option<&mut dyn NodeImageProgressSink>,
|
progress: Option<&mut dyn NodeImageProgressSink>,
|
||||||
) -> Result<PreparedNodeImage, String> {
|
) -> Result<String, String> {
|
||||||
prepare_node_image_with_progress(request, progress)
|
prepare_node_image_with_progress(request, progress)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1525,7 +1603,7 @@ fn prepare_runtime(config: &Config) -> Result<String, String> {
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
fn prepare_runtime_with<F>(config: &Config, prepare_node_image_fn: F) -> Result<String, String>
|
fn prepare_runtime_with<F>(config: &Config, prepare_node_image_fn: F) -> Result<String, String>
|
||||||
where
|
where
|
||||||
F: FnMut(NodeImageRequest) -> Result<PreparedNodeImage, String>,
|
F: FnMut(NodeImageRequest) -> Result<String, String>,
|
||||||
{
|
{
|
||||||
let mut prepare_node_image_fn = prepare_node_image_fn;
|
let mut prepare_node_image_fn = prepare_node_image_fn;
|
||||||
prepare_runtime_with_progress(
|
prepare_runtime_with_progress(
|
||||||
|
|
@ -1541,10 +1619,7 @@ fn prepare_runtime_with_progress<F>(
|
||||||
progress: Option<&mut ChatDatastream>,
|
progress: Option<&mut ChatDatastream>,
|
||||||
) -> Result<String, String>
|
) -> Result<String, String>
|
||||||
where
|
where
|
||||||
F: FnMut(
|
F: FnMut(NodeImageRequest, Option<&mut dyn NodeImageProgressSink>) -> Result<String, String>,
|
||||||
NodeImageRequest,
|
|
||||||
Option<&mut dyn NodeImageProgressSink>,
|
|
||||||
) -> Result<PreparedNodeImage, String>,
|
|
||||||
{
|
{
|
||||||
let mut progress = progress;
|
let mut progress = progress;
|
||||||
let binary_mode = if config.skip_rebuild {
|
let binary_mode = if config.skip_rebuild {
|
||||||
|
|
@ -1758,9 +1833,9 @@ where
|
||||||
CHAT_RUNTIME_CHANNEL,
|
CHAT_RUNTIME_CHANNEL,
|
||||||
"prepare_node_image",
|
"prepare_node_image",
|
||||||
"ready",
|
"ready",
|
||||||
json!({"provider": config.provider.as_str(), "command_label": "prepare_node_image", "image_ref": prepared.image_ref, "elapsed_ms": prepare_node_image_started.elapsed().as_millis()}),
|
json!({"provider": config.provider.as_str(), "command_label": "prepare_node_image", "image_ref": &prepared, "elapsed_ms": prepare_node_image_started.elapsed().as_millis()}),
|
||||||
);
|
);
|
||||||
Ok(prepared.image_ref)
|
Ok(prepared)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stdin_prompt_events() -> mpsc::Receiver<PromptInput> {
|
fn stdin_prompt_events() -> mpsc::Receiver<PromptInput> {
|
||||||
|
|
@ -1852,32 +1927,23 @@ fn run_chat_loop_with_input_and_progress(
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
fn run_chat_session_with_output<R, W, O>(
|
fn run_chat_session_with_output(
|
||||||
writer: &mut W,
|
writer: &mut impl Write,
|
||||||
reader: R,
|
reader: impl BufRead,
|
||||||
input_rx: mpsc::Receiver<PromptInput>,
|
input_rx: mpsc::Receiver<PromptInput>,
|
||||||
max_tokens: u32,
|
max_tokens: u32,
|
||||||
output: &mut O,
|
output: &mut impl Write,
|
||||||
) -> Result<(), String>
|
) -> Result<(), String> {
|
||||||
where
|
|
||||||
R: BufRead,
|
|
||||||
W: Write,
|
|
||||||
O: Write,
|
|
||||||
{
|
|
||||||
run_chat_session_with_output_and_progress(writer, reader, input_rx, max_tokens, output, None)
|
run_chat_session_with_output_and_progress(writer, reader, input_rx, max_tokens, output, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run_chat_session_with_progress<R, W>(
|
fn run_chat_session_with_progress(
|
||||||
writer: &mut W,
|
writer: &mut impl Write,
|
||||||
reader: R,
|
reader: impl BufRead,
|
||||||
input_rx: mpsc::Receiver<PromptInput>,
|
input_rx: mpsc::Receiver<PromptInput>,
|
||||||
max_tokens: u32,
|
max_tokens: u32,
|
||||||
progress: Option<&mut ChatDatastream>,
|
progress: Option<&mut ChatDatastream>,
|
||||||
) -> Result<(), String>
|
) -> Result<(), String> {
|
||||||
where
|
|
||||||
R: BufRead,
|
|
||||||
W: Write,
|
|
||||||
{
|
|
||||||
let mut output = io::stdout();
|
let mut output = io::stdout();
|
||||||
run_chat_session_with_output_and_progress(
|
run_chat_session_with_output_and_progress(
|
||||||
writer,
|
writer,
|
||||||
|
|
@ -1905,19 +1971,14 @@ fn prompt_hash_hex(prompt: &str) -> String {
|
||||||
blake3::hash(prompt.as_bytes()).to_hex().to_string()
|
blake3::hash(prompt.as_bytes()).to_hex().to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run_chat_session_with_output_and_progress<R, W, O>(
|
fn run_chat_session_with_output_and_progress(
|
||||||
writer: &mut W,
|
writer: &mut impl Write,
|
||||||
mut reader: R,
|
mut reader: impl BufRead,
|
||||||
input_rx: mpsc::Receiver<PromptInput>,
|
input_rx: mpsc::Receiver<PromptInput>,
|
||||||
max_tokens: u32,
|
max_tokens: u32,
|
||||||
output: &mut O,
|
output: &mut impl Write,
|
||||||
progress: Option<&mut ChatDatastream>,
|
progress: Option<&mut ChatDatastream>,
|
||||||
) -> Result<(), String>
|
) -> Result<(), String> {
|
||||||
where
|
|
||||||
R: BufRead,
|
|
||||||
W: Write,
|
|
||||||
O: Write,
|
|
||||||
{
|
|
||||||
let mut progress = progress;
|
let mut progress = progress;
|
||||||
let mut next_request_id = 1_u64;
|
let mut next_request_id = 1_u64;
|
||||||
let mut next_prompt_index = 1_u64;
|
let mut next_prompt_index = 1_u64;
|
||||||
|
|
@ -2375,10 +2436,8 @@ fn parse_pipeline_stages_value(
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
use std::ffi::{OsStr, OsString};
|
use std::ffi::OsString;
|
||||||
use std::io::{Cursor, Read};
|
use std::io::{Cursor, Read};
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
use std::os::unix::process::CommandExt;
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
|
use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
|
||||||
|
|
@ -2487,50 +2546,6 @@ mod tests {
|
||||||
path
|
path
|
||||||
}
|
}
|
||||||
|
|
||||||
fn base_config(provider: ProviderKind) -> Config {
|
|
||||||
Config {
|
|
||||||
orch_bin: PathBuf::from("/tmp/mvp-orchestrator"),
|
|
||||||
worker_bin: PathBuf::from("/tmp/mvp-worker-node"),
|
|
||||||
rpc_addr: DEFAULT_RPC_ADDR.to_owned(),
|
|
||||||
node_image: "docker.io/acme/node:latest".to_owned(),
|
|
||||||
provider,
|
|
||||||
image_tag: None,
|
|
||||||
cached_model: None,
|
|
||||||
datastream_frame_log: None,
|
|
||||||
run_id: 1,
|
|
||||||
vastai_yes: false,
|
|
||||||
vastai: None,
|
|
||||||
model: ChatModelConfig::default(),
|
|
||||||
pipeline_stages: 1,
|
|
||||||
max_tokens: DEFAULT_MAX_TOKENS,
|
|
||||||
skip_rebuild: true,
|
|
||||||
gpu_run: false,
|
|
||||||
relay_mode: None,
|
|
||||||
relay_url: None,
|
|
||||||
endpoint_addr_mask: EndpointAddrMask::Full,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn valid_vastai() -> ResolvedVastAiConfig {
|
|
||||||
ResolvedVastAiConfig {
|
|
||||||
api_key: "secret".to_owned(),
|
|
||||||
relay_url: "https://relay.example".to_owned(),
|
|
||||||
image: "docker.io/acme/node:latest".to_owned(),
|
|
||||||
bootstrap_command: "boot".to_owned(),
|
|
||||||
disk_gb: None,
|
|
||||||
gpu_name: None,
|
|
||||||
min_gpu_ram_mb: None,
|
|
||||||
min_down_mbps: None,
|
|
||||||
min_up_mbps: None,
|
|
||||||
max_dph_total: None,
|
|
||||||
min_reliability: None,
|
|
||||||
require_verified: None,
|
|
||||||
blacklist_hosts: Vec::new(),
|
|
||||||
onstart: None,
|
|
||||||
ssh_identity: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn channel_lines(lines: &[&str]) -> mpsc::Receiver<PromptInput> {
|
fn channel_lines(lines: &[&str]) -> mpsc::Receiver<PromptInput> {
|
||||||
let (tx, rx) = mpsc::channel();
|
let (tx, rx) = mpsc::channel();
|
||||||
for line in lines {
|
for line in lines {
|
||||||
|
|
@ -2802,131 +2817,6 @@ kind = "mock"
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
struct MockApproval {
|
|
||||||
terminal: bool,
|
|
||||||
answer: Result<bool, String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl VastAiApproval for MockApproval {
|
|
||||||
fn stdin_is_terminal(&self) -> bool {
|
|
||||||
self.terminal
|
|
||||||
}
|
|
||||||
|
|
||||||
fn ask(&mut self) -> Result<bool, String> {
|
|
||||||
self.answer.clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn panic_prepare_node_image(_: NodeImageRequest) -> Result<PreparedNodeImage, String> {
|
|
||||||
panic!("image preparer must not be called when --skip-rebuild is set")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn panic_prepare_node_image_with_progress(
|
|
||||||
_request: NodeImageRequest,
|
|
||||||
_progress: Option<&mut dyn NodeImageProgressSink>,
|
|
||||||
) -> Result<PreparedNodeImage, String> {
|
|
||||||
panic!("image preparer must not be called when --skip-rebuild is set")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn runtime_events(path: &Path) -> Vec<serde_json::Value> {
|
|
||||||
fs::read_to_string(path)
|
|
||||||
.expect("read progress archive")
|
|
||||||
.lines()
|
|
||||||
.filter_map(|line| {
|
|
||||||
let outer: serde_json::Value = serde_json::from_str(line).ok()?;
|
|
||||||
if outer.get("channel").and_then(serde_json::Value::as_str)
|
|
||||||
!= Some(CHAT_RUNTIME_CHANNEL)
|
|
||||||
{
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
outer
|
|
||||||
.get("payload")?
|
|
||||||
.get("value")?
|
|
||||||
.as_str()
|
|
||||||
.and_then(|text| serde_json::from_str::<serde_json::Value>(text).ok())
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emit_fake_node_image_progress(
|
|
||||||
progress: Option<&mut dyn NodeImageProgressSink>,
|
|
||||||
success: bool,
|
|
||||||
) {
|
|
||||||
let Some(sink) = progress else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
sink.emit(NodeImageProgressEvent {
|
|
||||||
command_label: None,
|
|
||||||
image_ref: Some("docker.io/acme/node:prepared".to_owned()),
|
|
||||||
elapsed_ms: None,
|
|
||||||
kind: NodeImageProgressEventKind::ImageReference {
|
|
||||||
role: "resolved".to_owned(),
|
|
||||||
image_ref: "docker.io/acme/node:prepared".to_owned(),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
sink.emit(NodeImageProgressEvent {
|
|
||||||
command_label: Some("build mvp node image".to_owned()),
|
|
||||||
image_ref: Some("docker.io/acme/node:prepared".to_owned()),
|
|
||||||
elapsed_ms: Some(0),
|
|
||||||
kind: NodeImageProgressEventKind::CommandStarted {
|
|
||||||
program: "fake-docker".to_owned(),
|
|
||||||
args: vec!["build".to_owned()],
|
|
||||||
},
|
|
||||||
});
|
|
||||||
sink.emit(NodeImageProgressEvent {
|
|
||||||
command_label: Some("build mvp node image".to_owned()),
|
|
||||||
image_ref: Some("docker.io/acme/node:prepared".to_owned()),
|
|
||||||
elapsed_ms: Some(1),
|
|
||||||
kind: NodeImageProgressEventKind::CommandStdout {
|
|
||||||
line: "building layer".to_owned(),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
sink.emit(NodeImageProgressEvent {
|
|
||||||
command_label: Some("build mvp node image".to_owned()),
|
|
||||||
image_ref: Some("docker.io/acme/node:prepared".to_owned()),
|
|
||||||
elapsed_ms: Some(2),
|
|
||||||
kind: NodeImageProgressEventKind::CommandStderr {
|
|
||||||
line: "pushing metadata".to_owned(),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
sink.emit(NodeImageProgressEvent {
|
|
||||||
command_label: Some("build mvp node image".to_owned()),
|
|
||||||
image_ref: Some("docker.io/acme/node:prepared".to_owned()),
|
|
||||||
elapsed_ms: Some(3),
|
|
||||||
kind: NodeImageProgressEventKind::CommandExited {
|
|
||||||
status: if success {
|
|
||||||
"exit status: 0".to_owned()
|
|
||||||
} else {
|
|
||||||
"exit status: 42".to_owned()
|
|
||||||
},
|
|
||||||
code: Some(if success { 0 } else { 42 }),
|
|
||||||
success,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
fn fake_prepare_node_image_with_progress(
|
|
||||||
_request: NodeImageRequest,
|
|
||||||
progress: Option<&mut dyn NodeImageProgressSink>,
|
|
||||||
) -> Result<PreparedNodeImage, String> {
|
|
||||||
emit_fake_node_image_progress(progress, true);
|
|
||||||
Ok(PreparedNodeImage {
|
|
||||||
image_ref: "docker.io/acme/node:prepared".to_owned(),
|
|
||||||
tag: "prepared".to_owned(),
|
|
||||||
already_available: false,
|
|
||||||
built: true,
|
|
||||||
pushed: false,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn failing_prepare_node_image_with_progress(
|
|
||||||
_request: NodeImageRequest,
|
|
||||||
progress: Option<&mut dyn NodeImageProgressSink>,
|
|
||||||
) -> Result<PreparedNodeImage, String> {
|
|
||||||
emit_fake_node_image_progress(progress, false);
|
|
||||||
Err("build mvp node image failed with exit status: 42".to_owned())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn prompt_loop_exits_cleanly_and_ignores_empty_prompts() {
|
fn prompt_loop_exits_cleanly_and_ignores_empty_prompts() {
|
||||||
let mut rpc_writer = Vec::new();
|
let mut rpc_writer = Vec::new();
|
||||||
|
|
|
||||||
|
|
@ -347,7 +347,89 @@ impl NodeAgentActor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn forward_prompt_or_snapshot(&mut self, ctx: &Ctx, msg: NodeAgentMsg) -> Option<NodeAgentMsg> {
|
||||||
|
match msg {
|
||||||
|
NodeAgentMsg::InferPrompt {
|
||||||
|
request_id,
|
||||||
|
prompt,
|
||||||
|
max_tokens,
|
||||||
|
reply_to,
|
||||||
|
} => {
|
||||||
|
if let Some(report_to) = self.report_to {
|
||||||
|
let _ = ctx.send(
|
||||||
|
report_to,
|
||||||
|
NodeAgentReport::PromptRequested {
|
||||||
|
request_id,
|
||||||
|
prompt,
|
||||||
|
max_tokens,
|
||||||
|
reply_to,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
NodeAgentMsg::EncodePrompt {
|
||||||
|
request_id,
|
||||||
|
prompt,
|
||||||
|
reply_to,
|
||||||
|
} => {
|
||||||
|
if let Some(report_to) = self.report_to {
|
||||||
|
let _ = ctx.send(
|
||||||
|
report_to,
|
||||||
|
NodeAgentReport::EncodePromptRequested {
|
||||||
|
request_id,
|
||||||
|
prompt,
|
||||||
|
reply_to,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
NodeAgentMsg::DecodeTokens {
|
||||||
|
request_id,
|
||||||
|
tokens,
|
||||||
|
reply_to,
|
||||||
|
} => {
|
||||||
|
if let Some(report_to) = self.report_to {
|
||||||
|
let _ = ctx.send(
|
||||||
|
report_to,
|
||||||
|
NodeAgentReport::DecodeTokensRequested {
|
||||||
|
request_id,
|
||||||
|
tokens,
|
||||||
|
reply_to,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
NodeAgentMsg::Snapshot { reply_to } => {
|
||||||
|
let _ = ctx.send(
|
||||||
|
reply_to,
|
||||||
|
NodeAgentReport::Snapshot {
|
||||||
|
commands: self
|
||||||
|
.core
|
||||||
|
.commands()
|
||||||
|
.iter()
|
||||||
|
.map(StageCommandWire::from)
|
||||||
|
.collect(),
|
||||||
|
events: self
|
||||||
|
.core
|
||||||
|
.events()
|
||||||
|
.iter()
|
||||||
|
.map(StageLifecycleWire::from)
|
||||||
|
.collect(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
other => Some(other),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn observe(&mut self, ctx: &Ctx, msg: NodeAgentMsg) {
|
fn observe(&mut self, ctx: &Ctx, msg: NodeAgentMsg) {
|
||||||
|
let Some(msg) = self.forward_prompt_or_snapshot(ctx, msg) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
match msg {
|
match msg {
|
||||||
NodeAgentMsg::ProvisionStage(provision) => {
|
NodeAgentMsg::ProvisionStage(provision) => {
|
||||||
self.core.observe(stage::StageEvent::ProvisionStage {
|
self.core.observe(stage::StageEvent::ProvisionStage {
|
||||||
|
|
@ -480,79 +562,10 @@ impl NodeAgentActor {
|
||||||
run_id: stage::RunId(run_id),
|
run_id: stage::RunId(run_id),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
NodeAgentMsg::InferPrompt {
|
NodeAgentMsg::InferPrompt { .. }
|
||||||
request_id,
|
| NodeAgentMsg::EncodePrompt { .. }
|
||||||
prompt,
|
| NodeAgentMsg::DecodeTokens { .. }
|
||||||
max_tokens,
|
| NodeAgentMsg::Snapshot { .. } => unreachable!("prompt messages returned early"),
|
||||||
reply_to,
|
|
||||||
} => {
|
|
||||||
if let Some(report_to) = self.report_to {
|
|
||||||
let _ = ctx.send(
|
|
||||||
report_to,
|
|
||||||
NodeAgentReport::PromptRequested {
|
|
||||||
request_id,
|
|
||||||
prompt,
|
|
||||||
max_tokens,
|
|
||||||
reply_to,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
NodeAgentMsg::EncodePrompt {
|
|
||||||
request_id,
|
|
||||||
prompt,
|
|
||||||
reply_to,
|
|
||||||
} => {
|
|
||||||
if let Some(report_to) = self.report_to {
|
|
||||||
let _ = ctx.send(
|
|
||||||
report_to,
|
|
||||||
NodeAgentReport::EncodePromptRequested {
|
|
||||||
request_id,
|
|
||||||
prompt,
|
|
||||||
reply_to,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
NodeAgentMsg::DecodeTokens {
|
|
||||||
request_id,
|
|
||||||
tokens,
|
|
||||||
reply_to,
|
|
||||||
} => {
|
|
||||||
if let Some(report_to) = self.report_to {
|
|
||||||
let _ = ctx.send(
|
|
||||||
report_to,
|
|
||||||
NodeAgentReport::DecodeTokensRequested {
|
|
||||||
request_id,
|
|
||||||
tokens,
|
|
||||||
reply_to,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
NodeAgentMsg::Snapshot { reply_to } => {
|
|
||||||
let _ = ctx.send(
|
|
||||||
reply_to,
|
|
||||||
NodeAgentReport::Snapshot {
|
|
||||||
commands: self
|
|
||||||
.core
|
|
||||||
.commands()
|
|
||||||
.iter()
|
|
||||||
.map(StageCommandWire::from)
|
|
||||||
.collect(),
|
|
||||||
events: self
|
|
||||||
.core
|
|
||||||
.events()
|
|
||||||
.iter()
|
|
||||||
.map(StageLifecycleWire::from)
|
|
||||||
.collect(),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
self.drain_outputs(ctx);
|
self.drain_outputs(ctx);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1252,6 +1252,25 @@ impl WorkerEdgeRuntime {
|
||||||
driver: &mut IrohDriver,
|
driver: &mut IrohDriver,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
loop {
|
loop {
|
||||||
|
let progressed =
|
||||||
|
self.drain_edge_commands(worker, arena_manager, config, datastream, driver)?
|
||||||
|
|| self.drain_driver_events()
|
||||||
|
|| self.drain_edge_events(stack, node_actor)?;
|
||||||
|
if !progressed {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain_edge_commands(
|
||||||
|
&mut self,
|
||||||
|
worker: &mut TinygradWorker,
|
||||||
|
arena_manager: &Arc<Mutex<arena::ArenaManager>>,
|
||||||
|
config: &DeploymentConfig,
|
||||||
|
datastream: &mut NodeDatastream,
|
||||||
|
driver: &mut IrohDriver,
|
||||||
|
) -> Result<bool, String> {
|
||||||
let mut progressed = false;
|
let mut progressed = false;
|
||||||
while self.edge_command_cursor < self.establisher.commands().len() {
|
while self.edge_command_cursor < self.establisher.commands().len() {
|
||||||
let command = self.establisher.commands()[self.edge_command_cursor].clone();
|
let command = self.establisher.commands()[self.edge_command_cursor].clone();
|
||||||
|
|
@ -1457,7 +1476,11 @@ impl WorkerEdgeRuntime {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Ok(progressed)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain_driver_events(&mut self) -> bool {
|
||||||
|
let mut progressed = false;
|
||||||
while self.driver_event_cursor < self.driver_model.events().len() {
|
while self.driver_event_cursor < self.driver_model.events().len() {
|
||||||
let event = self.driver_model.events()[self.driver_event_cursor].clone();
|
let event = self.driver_model.events()[self.driver_event_cursor].clone();
|
||||||
self.driver_event_cursor += 1;
|
self.driver_event_cursor += 1;
|
||||||
|
|
@ -1494,7 +1517,15 @@ impl WorkerEdgeRuntime {
|
||||||
driver_model::DriverEventOut::StreamClosed { .. } => {}
|
driver_model::DriverEventOut::StreamClosed { .. } => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
progressed
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain_edge_events(
|
||||||
|
&mut self,
|
||||||
|
stack: &DistributionRuntimeStack,
|
||||||
|
node_actor: ActorAddress,
|
||||||
|
) -> Result<bool, String> {
|
||||||
|
let mut progressed = false;
|
||||||
while self.edge_event_cursor < self.establisher.events().len() {
|
while self.edge_event_cursor < self.establisher.events().len() {
|
||||||
let event = self.establisher.events()[self.edge_event_cursor].clone();
|
let event = self.establisher.events()[self.edge_event_cursor].clone();
|
||||||
self.edge_event_cursor += 1;
|
self.edge_event_cursor += 1;
|
||||||
|
|
@ -1543,11 +1574,7 @@ impl WorkerEdgeRuntime {
|
||||||
edge::EdgeLifecycleEvent::EdgeStopped { .. } => {}
|
edge::EdgeLifecycleEvent::EdgeStopped { .. } => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !progressed {
|
Ok(progressed)
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -20,8 +20,7 @@ pub mod runtime_stack;
|
||||||
pub mod workload;
|
pub mod workload;
|
||||||
|
|
||||||
pub use crate::run_plan::NodeId;
|
pub use crate::run_plan::NodeId;
|
||||||
pub use engine::{ClusterBuilder, ClusterHandle};
|
pub use engine::ClusterBuilder;
|
||||||
pub use error::{EngineBuildError, PlanningError};
|
|
||||||
pub use events::EngineEvent;
|
pub use events::EngineEvent;
|
||||||
pub use launcher::StaticNodeLauncher;
|
pub use launcher::StaticNodeLauncher;
|
||||||
pub use model::{DTypeFamily, ModelArtifact, ModelSpec};
|
pub use model::{DTypeFamily, ModelArtifact, ModelSpec};
|
||||||
|
|
@ -29,5 +28,3 @@ pub use node_image::{NodeImageSpec, WorkerRuntimeSpec};
|
||||||
pub use planner::FixedLinearPipelinePlanner;
|
pub use planner::FixedLinearPipelinePlanner;
|
||||||
pub use pool::{NodeCapability, NodeLease, ResourceFacts, StaticPoolProvider};
|
pub use pool::{NodeCapability, NodeLease, ResourceFacts, StaticPoolProvider};
|
||||||
pub use roles::RoleKind;
|
pub use roles::RoleKind;
|
||||||
pub use runtime_stack::RuntimeNode;
|
|
||||||
pub use workload::WorkloadAdapter;
|
|
||||||
|
|
|
||||||
|
|
@ -230,14 +230,7 @@ impl OrchestratorRun {
|
||||||
RunEvent::StageReady {
|
RunEvent::StageReady {
|
||||||
run_id,
|
run_id,
|
||||||
stage_index,
|
stage_index,
|
||||||
} => {
|
} => self.stage_ready(run_id, stage_index),
|
||||||
if run_id != self.config.run_id || !self.plan_has_stage(stage_index) {
|
|
||||||
self.fault(RunFaultReason::UnknownStageReady { stage_index });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
self.ready_stages.insert(stage_index);
|
|
||||||
self.maybe_inject_initial();
|
|
||||||
}
|
|
||||||
RunEvent::TokenInEndpointReady => {
|
RunEvent::TokenInEndpointReady => {
|
||||||
self.token_in_ready = true;
|
self.token_in_ready = true;
|
||||||
self.maybe_inject_initial();
|
self.maybe_inject_initial();
|
||||||
|
|
@ -250,11 +243,52 @@ impl OrchestratorRun {
|
||||||
sequence,
|
sequence,
|
||||||
token_id,
|
token_id,
|
||||||
eos,
|
eos,
|
||||||
|
} => self.token_received(sequence, token_id, eos),
|
||||||
|
RunEvent::StageFault { run_id, .. }
|
||||||
|
| RunEvent::EndpointFault { run_id, .. }
|
||||||
|
| RunEvent::OperatorStop { run_id }
|
||||||
|
| RunEvent::MembershipLost { run_id, .. }
|
||||||
|
| RunEvent::StageStopped { run_id, .. }
|
||||||
|
if run_id != self.config.run_id => {}
|
||||||
|
RunEvent::StageFault {
|
||||||
|
stage_index,
|
||||||
|
reason,
|
||||||
|
..
|
||||||
} => {
|
} => {
|
||||||
if self.terminal {
|
self.fault(RunFaultReason::StageFault {
|
||||||
|
stage_index,
|
||||||
|
reason,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
RunEvent::EndpointFault { endpoint, .. } => {
|
||||||
|
self.fault(RunFaultReason::EndpointFault { endpoint });
|
||||||
|
}
|
||||||
|
RunEvent::OperatorStop { .. } => self.operator_stop(),
|
||||||
|
RunEvent::MembershipLost { node_id, .. } => {
|
||||||
|
self.fault(RunFaultReason::MembershipLost { node_id });
|
||||||
|
}
|
||||||
|
RunEvent::StageStopped { stage_index, .. } => {
|
||||||
|
self.stopped_stages.insert(stage_index);
|
||||||
|
self.maybe_torn_down();
|
||||||
|
}
|
||||||
|
RunEvent::TokenEndpointsStopped => {
|
||||||
|
self.token_endpoints_stopped = true;
|
||||||
|
self.maybe_torn_down();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stage_ready(&mut self, run_id: RunId, stage_index: u32) {
|
||||||
|
if run_id != self.config.run_id || !self.plan_has_stage(stage_index) {
|
||||||
|
self.fault(RunFaultReason::UnknownStageReady { stage_index });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if sequence != self.expected_token_sequence {
|
self.ready_stages.insert(stage_index);
|
||||||
|
self.maybe_inject_initial();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn token_received(&mut self, sequence: u64, token_id: u32, eos: bool) {
|
||||||
|
if self.terminal || sequence != self.expected_token_sequence {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.expected_token_sequence += 1;
|
self.expected_token_sequence += 1;
|
||||||
|
|
@ -266,43 +300,6 @@ impl OrchestratorRun {
|
||||||
self.complete();
|
self.complete();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
RunEvent::StageFault {
|
|
||||||
run_id,
|
|
||||||
stage_index,
|
|
||||||
reason,
|
|
||||||
} if run_id == self.config.run_id => {
|
|
||||||
self.fault(RunFaultReason::StageFault {
|
|
||||||
stage_index,
|
|
||||||
reason,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
RunEvent::EndpointFault { run_id, endpoint } if run_id == self.config.run_id => {
|
|
||||||
self.fault(RunFaultReason::EndpointFault { endpoint });
|
|
||||||
}
|
|
||||||
RunEvent::OperatorStop { run_id } if run_id == self.config.run_id => {
|
|
||||||
self.operator_stop();
|
|
||||||
}
|
|
||||||
RunEvent::MembershipLost { run_id, node_id } if run_id == self.config.run_id => {
|
|
||||||
self.fault(RunFaultReason::MembershipLost { node_id });
|
|
||||||
}
|
|
||||||
RunEvent::StageStopped {
|
|
||||||
run_id,
|
|
||||||
stage_index,
|
|
||||||
} if run_id == self.config.run_id => {
|
|
||||||
self.stopped_stages.insert(stage_index);
|
|
||||||
self.maybe_torn_down();
|
|
||||||
}
|
|
||||||
RunEvent::TokenEndpointsStopped => {
|
|
||||||
self.token_endpoints_stopped = true;
|
|
||||||
self.maybe_torn_down();
|
|
||||||
}
|
|
||||||
RunEvent::StageFault { .. }
|
|
||||||
| RunEvent::EndpointFault { .. }
|
|
||||||
| RunEvent::OperatorStop { .. }
|
|
||||||
| RunEvent::MembershipLost { .. }
|
|
||||||
| RunEvent::StageStopped { .. } => {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn advance_time_ms(&mut self, _delta: u64) {}
|
pub fn advance_time_ms(&mut self, _delta: u64) {}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -302,22 +302,10 @@ impl StageController {
|
||||||
StageEvent::WorkerReady => self.worker_ready = true,
|
StageEvent::WorkerReady => self.worker_ready = true,
|
||||||
StageEvent::WeightsReady => self.weights_ready = true,
|
StageEvent::WeightsReady => self.weights_ready = true,
|
||||||
StageEvent::InboundEdgeReady { edge_id } => {
|
StageEvent::InboundEdgeReady { edge_id } => {
|
||||||
if self
|
self.edge_ready(edge_id, EdgeDirection::Inbound)
|
||||||
.provision
|
|
||||||
.as_ref()
|
|
||||||
.is_some_and(|p| p.inbound.edge_id == edge_id)
|
|
||||||
{
|
|
||||||
self.inbound_ready = true;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
StageEvent::OutboundEdgeReady { edge_id } => {
|
StageEvent::OutboundEdgeReady { edge_id } => {
|
||||||
if self
|
self.edge_ready(edge_id, EdgeDirection::Outbound)
|
||||||
.provision
|
|
||||||
.as_ref()
|
|
||||||
.is_some_and(|p| p.outbound.edge_id == edge_id)
|
|
||||||
{
|
|
||||||
self.outbound_ready = true;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
StageEvent::ObjectLoaded {
|
StageEvent::ObjectLoaded {
|
||||||
edge_id,
|
edge_id,
|
||||||
|
|
@ -340,6 +328,21 @@ impl StageController {
|
||||||
self.maybe_stage_ready();
|
self.maybe_stage_ready();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn edge_ready(&mut self, edge_id: EdgeId, direction: EdgeDirection) {
|
||||||
|
let Some(provision) = &self.provision else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
match direction {
|
||||||
|
EdgeDirection::Inbound if provision.inbound.edge_id == edge_id => {
|
||||||
|
self.inbound_ready = true
|
||||||
|
}
|
||||||
|
EdgeDirection::Outbound if provision.outbound.edge_id == edge_id => {
|
||||||
|
self.outbound_ready = true
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn commands(&self) -> &[StageCommand] {
|
pub fn commands(&self) -> &[StageCommand] {
|
||||||
&self.commands
|
&self.commands
|
||||||
}
|
}
|
||||||
|
|
@ -380,16 +383,18 @@ impl StageController {
|
||||||
if self.stage_ready_emitted || self.faulted || self.stopped || self.stopping_run.is_some() {
|
if self.stage_ready_emitted || self.faulted || self.stopped || self.stopping_run.is_some() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if self.worker_ready && self.weights_ready && self.inbound_ready && self.outbound_ready {
|
if !(self.worker_ready && self.weights_ready && self.inbound_ready && self.outbound_ready) {
|
||||||
if let Some(provision) = &self.provision {
|
return;
|
||||||
|
}
|
||||||
|
let Some(provision) = &self.provision else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
self.stage_ready_emitted = true;
|
self.stage_ready_emitted = true;
|
||||||
self.events.push(StageLifecycleEvent::StageReady {
|
self.events.push(StageLifecycleEvent::StageReady {
|
||||||
run_id: provision.run_id,
|
run_id: provision.run_id,
|
||||||
stage_index: provision.stage_index,
|
stage_index: provision.stage_index,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn object_loaded(
|
fn object_loaded(
|
||||||
&mut self,
|
&mut self,
|
||||||
|
|
|
||||||
|
|
@ -174,23 +174,26 @@ enum GgufValueType {
|
||||||
|
|
||||||
impl GgufValueType {
|
impl GgufValueType {
|
||||||
fn read<R: Read>(reader: &mut R) -> Result<Self, String> {
|
fn read<R: Read>(reader: &mut R) -> Result<Self, String> {
|
||||||
|
const VALUE_TYPES: [GgufValueType; 13] = [
|
||||||
|
GgufValueType::Uint8,
|
||||||
|
GgufValueType::Int8,
|
||||||
|
GgufValueType::Uint16,
|
||||||
|
GgufValueType::Int16,
|
||||||
|
GgufValueType::Uint32,
|
||||||
|
GgufValueType::Int32,
|
||||||
|
GgufValueType::Float32,
|
||||||
|
GgufValueType::Bool,
|
||||||
|
GgufValueType::String,
|
||||||
|
GgufValueType::Array,
|
||||||
|
GgufValueType::Uint64,
|
||||||
|
GgufValueType::Int64,
|
||||||
|
GgufValueType::Float64,
|
||||||
|
];
|
||||||
let raw = read_u32(reader)?;
|
let raw = read_u32(reader)?;
|
||||||
match raw {
|
VALUE_TYPES
|
||||||
0 => Ok(Self::Uint8),
|
.get(raw as usize)
|
||||||
1 => Ok(Self::Int8),
|
.copied()
|
||||||
2 => Ok(Self::Uint16),
|
.ok_or_else(|| format!("unsupported GGUF metadata value type {raw}"))
|
||||||
3 => Ok(Self::Int16),
|
|
||||||
4 => Ok(Self::Uint32),
|
|
||||||
5 => Ok(Self::Int32),
|
|
||||||
6 => Ok(Self::Float32),
|
|
||||||
7 => Ok(Self::Bool),
|
|
||||||
8 => Ok(Self::String),
|
|
||||||
9 => Ok(Self::Array),
|
|
||||||
10 => Ok(Self::Uint64),
|
|
||||||
11 => Ok(Self::Int64),
|
|
||||||
12 => Ok(Self::Float64),
|
|
||||||
other => Err(format!("unsupported GGUF metadata value type {other}")),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_integer(self) -> bool {
|
fn is_integer(self) -> bool {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::{Read, Seek, SeekFrom, Write};
|
use std::io::{Read, Seek, SeekFrom, Write};
|
||||||
use std::path::Path;
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
|
@ -537,54 +537,16 @@ where
|
||||||
}
|
}
|
||||||
let metadata_body = &metadata_prefix[24..];
|
let metadata_body = &metadata_prefix[24..];
|
||||||
let alignment = u64::from(plan.alignment.max(1));
|
let alignment = u64::from(plan.alignment.max(1));
|
||||||
let mut data_offsets = Vec::with_capacity(plan.tensors.len());
|
let data_offsets = stage_shard_data_offsets(plan, alignment)?;
|
||||||
let mut data_cursor = 0_u64;
|
|
||||||
for tensor in &plan.tensors {
|
|
||||||
data_cursor = align_to(data_cursor, alignment)?;
|
|
||||||
data_offsets.push(data_cursor);
|
|
||||||
data_cursor = data_cursor
|
|
||||||
.checked_add(tensor.byte_len)
|
|
||||||
.ok_or_else(|| format!("stage shard data size overflow at tensor {}", tensor.name))?;
|
|
||||||
}
|
|
||||||
|
|
||||||
let partial_path = output_path.with_extension(format!(
|
let partial_path = partial_stage_shard_path(output_path);
|
||||||
"{}partial",
|
|
||||||
output_path
|
|
||||||
.extension()
|
|
||||||
.and_then(|value| value.to_str())
|
|
||||||
.map(|ext| format!("{ext}."))
|
|
||||||
.unwrap_or_default()
|
|
||||||
));
|
|
||||||
if let Some(parent) = output_path.parent() {
|
if let Some(parent) = output_path.parent() {
|
||||||
std::fs::create_dir_all(parent)
|
std::fs::create_dir_all(parent)
|
||||||
.map_err(|e| format!("create stage shard cache dir {}: {e}", parent.display()))?;
|
.map_err(|e| format!("create stage shard cache dir {}: {e}", parent.display()))?;
|
||||||
}
|
}
|
||||||
let mut out = File::create(&partial_path)
|
let mut out = File::create(&partial_path)
|
||||||
.map_err(|e| format!("create stage shard {}: {e}", partial_path.display()))?;
|
.map_err(|e| format!("create stage shard {}: {e}", partial_path.display()))?;
|
||||||
out.write_all(GGUF_MAGIC)
|
write_stage_shard_header(&mut out, plan, metadata_body, &data_offsets, alignment)?;
|
||||||
.map_err(|e| format!("write stage shard magic: {e}"))?;
|
|
||||||
out.write_all(&SUPPORTED_GGUF_VERSION.to_le_bytes())
|
|
||||||
.map_err(|e| format!("write stage shard version: {e}"))?;
|
|
||||||
out.write_all(&(plan.tensors.len() as u64).to_le_bytes())
|
|
||||||
.map_err(|e| format!("write stage shard tensor count: {e}"))?;
|
|
||||||
out.write_all(&plan.metadata_count.to_le_bytes())
|
|
||||||
.map_err(|e| format!("write stage shard metadata count: {e}"))?;
|
|
||||||
out.write_all(metadata_body)
|
|
||||||
.map_err(|e| format!("write stage shard metadata: {e}"))?;
|
|
||||||
for (tensor, data_offset) in plan.tensors.iter().zip(data_offsets.iter().copied()) {
|
|
||||||
write_gguf_string(&mut out, &tensor.name)?;
|
|
||||||
out.write_all(&(tensor.dims.len() as u32).to_le_bytes())
|
|
||||||
.map_err(|e| format!("write tensor dim count for {}: {e}", tensor.name))?;
|
|
||||||
for dim in &tensor.dims {
|
|
||||||
out.write_all(&dim.to_le_bytes())
|
|
||||||
.map_err(|e| format!("write tensor dim for {}: {e}", tensor.name))?;
|
|
||||||
}
|
|
||||||
out.write_all(&tensor.ggml_type.to_le_bytes())
|
|
||||||
.map_err(|e| format!("write tensor type for {}: {e}", tensor.name))?;
|
|
||||||
out.write_all(&data_offset.to_le_bytes())
|
|
||||||
.map_err(|e| format!("write tensor offset for {}: {e}", tensor.name))?;
|
|
||||||
}
|
|
||||||
pad_writer_to_alignment(&mut out, alignment)?;
|
|
||||||
let mut written_data = 0_u64;
|
let mut written_data = 0_u64;
|
||||||
let mut tensor_index = 0_usize;
|
let mut tensor_index = 0_usize;
|
||||||
for range in &plan.merged_tensor_ranges {
|
for range in &plan.merged_tensor_ranges {
|
||||||
|
|
@ -712,6 +674,63 @@ where
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn stage_shard_data_offsets(plan: &StageShardPlan, alignment: u64) -> Result<Vec<u64>, String> {
|
||||||
|
let mut offsets = Vec::with_capacity(plan.tensors.len());
|
||||||
|
let mut cursor = 0_u64;
|
||||||
|
for tensor in &plan.tensors {
|
||||||
|
cursor = align_to(cursor, alignment)?;
|
||||||
|
offsets.push(cursor);
|
||||||
|
cursor = cursor
|
||||||
|
.checked_add(tensor.byte_len)
|
||||||
|
.ok_or_else(|| format!("stage shard data size overflow at tensor {}", tensor.name))?;
|
||||||
|
}
|
||||||
|
Ok(offsets)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn partial_stage_shard_path(output_path: &Path) -> PathBuf {
|
||||||
|
output_path.with_extension(format!(
|
||||||
|
"{}partial",
|
||||||
|
output_path
|
||||||
|
.extension()
|
||||||
|
.and_then(|value| value.to_str())
|
||||||
|
.map(|ext| format!("{ext}."))
|
||||||
|
.unwrap_or_default()
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_stage_shard_header(
|
||||||
|
out: &mut File,
|
||||||
|
plan: &StageShardPlan,
|
||||||
|
metadata_body: &[u8],
|
||||||
|
data_offsets: &[u64],
|
||||||
|
alignment: u64,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
out.write_all(GGUF_MAGIC)
|
||||||
|
.map_err(|e| format!("write stage shard magic: {e}"))?;
|
||||||
|
out.write_all(&SUPPORTED_GGUF_VERSION.to_le_bytes())
|
||||||
|
.map_err(|e| format!("write stage shard version: {e}"))?;
|
||||||
|
out.write_all(&(plan.tensors.len() as u64).to_le_bytes())
|
||||||
|
.map_err(|e| format!("write stage shard tensor count: {e}"))?;
|
||||||
|
out.write_all(&plan.metadata_count.to_le_bytes())
|
||||||
|
.map_err(|e| format!("write stage shard metadata count: {e}"))?;
|
||||||
|
out.write_all(metadata_body)
|
||||||
|
.map_err(|e| format!("write stage shard metadata: {e}"))?;
|
||||||
|
for (tensor, data_offset) in plan.tensors.iter().zip(data_offsets.iter().copied()) {
|
||||||
|
write_gguf_string(out, &tensor.name)?;
|
||||||
|
out.write_all(&(tensor.dims.len() as u32).to_le_bytes())
|
||||||
|
.map_err(|e| format!("write tensor dim count for {}: {e}", tensor.name))?;
|
||||||
|
for dim in &tensor.dims {
|
||||||
|
out.write_all(&dim.to_le_bytes())
|
||||||
|
.map_err(|e| format!("write tensor dim for {}: {e}", tensor.name))?;
|
||||||
|
}
|
||||||
|
out.write_all(&tensor.ggml_type.to_le_bytes())
|
||||||
|
.map_err(|e| format!("write tensor type for {}: {e}", tensor.name))?;
|
||||||
|
out.write_all(&data_offset.to_le_bytes())
|
||||||
|
.map_err(|e| format!("write tensor offset for {}: {e}", tensor.name))?;
|
||||||
|
}
|
||||||
|
pad_writer_to_alignment(out, alignment)
|
||||||
|
}
|
||||||
|
|
||||||
fn fetch_http_range(url: &str, start: u64, len: u64) -> Result<Vec<u8>, String> {
|
fn fetch_http_range(url: &str, start: u64, len: u64) -> Result<Vec<u8>, String> {
|
||||||
if len == 0 {
|
if len == 0 {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
|
|
@ -793,22 +812,26 @@ enum GgufValueType {
|
||||||
|
|
||||||
impl GgufValueType {
|
impl GgufValueType {
|
||||||
fn read<R: Read>(reader: &mut R) -> Result<Self, String> {
|
fn read<R: Read>(reader: &mut R) -> Result<Self, String> {
|
||||||
match read_u32(reader)? {
|
const VALUE_TYPES: [GgufValueType; 13] = [
|
||||||
0 => Ok(Self::Uint8),
|
GgufValueType::Uint8,
|
||||||
1 => Ok(Self::Int8),
|
GgufValueType::Int8,
|
||||||
2 => Ok(Self::Uint16),
|
GgufValueType::Uint16,
|
||||||
3 => Ok(Self::Int16),
|
GgufValueType::Int16,
|
||||||
4 => Ok(Self::Uint32),
|
GgufValueType::Uint32,
|
||||||
5 => Ok(Self::Int32),
|
GgufValueType::Int32,
|
||||||
6 => Ok(Self::Float32),
|
GgufValueType::Float32,
|
||||||
7 => Ok(Self::Bool),
|
GgufValueType::Bool,
|
||||||
8 => Ok(Self::String),
|
GgufValueType::String,
|
||||||
9 => Ok(Self::Array),
|
GgufValueType::Array,
|
||||||
10 => Ok(Self::Uint64),
|
GgufValueType::Uint64,
|
||||||
11 => Ok(Self::Int64),
|
GgufValueType::Int64,
|
||||||
12 => Ok(Self::Float64),
|
GgufValueType::Float64,
|
||||||
other => Err(format!("unsupported GGUF value type {other}")),
|
];
|
||||||
}
|
let raw = read_u32(reader)?;
|
||||||
|
VALUE_TYPES
|
||||||
|
.get(raw as usize)
|
||||||
|
.copied()
|
||||||
|
.ok_or_else(|| format!("unsupported GGUF value type {raw}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fixed_width(self) -> Option<u64> {
|
fn fixed_width(self) -> Option<u64> {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue