feat(enhance): pick LoRAs from ComfyUI per job (#220)

Templates with `imageapi:lora-slot:<n>:high|low` nodes take LoRAs picked
per job from a catalog built live from ComfyUI.

- Catalog: ComfyUI's LoraLoaderModelOnly `lora_name` choices (cached
  60 s, re-read when a submit names an unknown id), paired by High/Low
  name tokens (StyleHigh/StyleLow, x-HIGH/x-LOW, x_high_noise/...). Files
  the workflow already loads outside the slots (lightx2v, switches) and
  `[template.catalog] exclude` substrings never show; ids derive from the
  names so they stay stable.
- POST /enhance `slots: [{lora, scale?}]` fills the slots in order: each
  pick sets the slot nodes' lora_name and scales their exported strength;
  unused slots run at 0, so the graph is never rewired. More picks than
  slots, duplicates or unknown ids are 400s.
- GET /enhance/templates gains `slots` and `catalog: {loras, strength,
  min, max}`; jobs store and return the picks (new enhance_jobs.lora_slots
  JSON column).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-11-03 16:28:23 -05:00
1 parent 6fffca9989
commit e2cf18678f
15 files changed
+1454 -33

No files matched your search

+281 -18
View File
@@ -18,6 +18,9 @@ pub const TITLE_PROMPT: &str = "imageapi:prompt";
pub const TITLE_REFERENCE: &str = "imageapi:reference";
/// `imageapi:lora:<id>`: a LoRA loader the app can switch off (#218).
pub const TITLE_LORA_PREFIX: &str = "imageapi:lora:";
/// `imageapi:lora-slot:<n>:high|low`: a LoRA loader the app fills with a
/// LoRA picked from the catalog (#220); unused slots run at strength 0.
pub const TITLE_SLOT_PREFIX: &str = "imageapi:lora-slot:";
/// LoRA loader inputs a switch scales (0 = off). `strength_model` is
/// required (LoraLoaderModelOnly); `strength_clip` is scaled when present
@@ -61,6 +64,11 @@ pub struct Template {
reference_id: Option<String>,
/// Declared LoRA switches that have nodes, in `enhance.toml` order.
loras: Vec<LoraSwitch>,
/// Pickable-LoRA slots, by slot number.
slots: Vec<Slot>,
/// `lora_name`s the workflow loads outside the slots (lightx2v, switch
/// LoRAs): never offered in the catalog.
fixed_loras: std::collections::HashSet<String>,
/// Image or video, from the save node's class.
pub output: OutputKind,
/// Startup warnings (stray output nodes).
@@ -75,6 +83,22 @@ struct LoraSwitch {
nodes: Vec<(String, Vec<(&'static str, f64)>)>,
}
/// One pickable-LoRA slot: its high- and low-noise loader nodes, each with
/// its exported strengths (a pick's scale multiplies them).
#[derive(Debug, Clone)]
struct Slot {
high: Vec<(String, Vec<(&'static str, f64)>)>,
low: Vec<(String, Vec<(&'static str, f64)>)>,
}
/// A LoRA picked for one slot: its two files and the resolved scale.
#[derive(Debug, Clone, PartialEq)]
pub struct SlotPick {
pub high: String,
pub low: String,
pub scale: f64,
}
/// Why a request can't be turned into a ComfyUI prompt. The message is for
/// the caller, so it names the problem in the user's terms.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -227,6 +251,69 @@ impl Template {
));
}
// Pickable-LoRA slots, and the LoRAs loaded outside them.
let exported = |id: &str| -> Vec<(&'static str, f64)> {
let inputs = inputs_of(&workflow, id);
LORA_STRENGTH_INPUTS
.iter()
.filter_map(|k| inputs.get(*k).and_then(Value::as_f64).map(|v| (*k, v)))
.collect()
};
let mut slot_map: BTreeMap<u32, Slot> = BTreeMap::new();
let mut fixed_loras = std::collections::HashSet::new();
for (id, node) in &workflow {
let Some(rest) = title_of(node).and_then(|t| t.strip_prefix(TITLE_SLOT_PREFIX)) else {
if let Some(name) = inputs_of(&workflow, id)
.get("lora_name")
.and_then(Value::as_str)
{
fixed_loras.insert(name.to_string());
}
continue;
};
let title = format!("{TITLE_SLOT_PREFIX}{rest}");
let (n, role) = rest
.split_once(':')
.and_then(|(n, role)| Some((n.parse::<u32>().ok().filter(|n| *n > 0)?, role)))
.ok_or_else(|| {
anyhow::anyhow!(
"node {id} titled {title:?}: expected {TITLE_SLOT_PREFIX}<n>:high or :low (n from 1)"
)
})?;
let inputs = inputs_of(&workflow, id);
if !inputs.get("lora_name").is_some_and(Value::is_string)
|| !inputs.get("strength_model").is_some_and(Value::is_number)
{
bail!(
"node {id} titled {title:?} needs a lora_name and a numeric strength_model \
(use a LoraLoaderModelOnly or LoraLoader)"
);
}
let slot = slot_map.entry(n).or_insert(Slot {
high: Vec::new(),
low: Vec::new(),
});
match role {
"high" => slot.high.push((id.clone(), exported(id))),
"low" => slot.low.push((id.clone(), exported(id))),
_ => bail!("node {id} titled {title:?}: the role must be high or low"),
}
}
for (n, slot) in &slot_map {
if slot.high.is_empty() || slot.low.is_empty() {
bail!("LoRA slot {n} needs both a :high and a :low node");
}
}
if let Some((last, _)) = slot_map.last_key_value()
&& *last as usize != slot_map.len()
{
bail!(
"LoRA slots must be numbered 1..{} without gaps",
slot_map.len()
);
}
let slots: Vec<Slot> = slot_map.into_values().collect();
for (id, node) in &workflow {
let class = node["class_type"].as_str().unwrap_or_default();
if STRAY_OUTPUT_CLASSES.contains(&class) {
@@ -248,6 +335,8 @@ impl Template {
prompt,
reference_id,
loras,
slots,
fixed_loras,
output,
warnings,
})
@@ -265,6 +354,16 @@ impl Template {
self.cfg.prompt_mode == PromptMode::Replace
}
/// How many pickable-LoRA slots the workflow has (#220).
pub fn slot_count(&self) -> usize {
self.slots.len()
}
/// `lora_name`s loaded outside the slots; kept out of the catalog.
pub fn fixed_loras(&self) -> &std::collections::HashSet<String> {
&self.fixed_loras
}
/// The LoRA switches clients can offer (declared and present in the
/// workflow), in `enhance.toml` order.
pub fn lora_switches(&self) -> impl Iterator<Item = &LoraConfig> {
@@ -288,8 +387,19 @@ impl Template {
prompt: Option<&str>,
reference: Option<&str>,
loras: &BTreeMap<String, LoraRequest>,
picks: &[SlotPick],
next_seed: &mut dyn FnMut() -> u64,
) -> Result<Filled, FillError> {
if picks.len() > self.slots.len() {
return Err(FillError(if self.slots.is_empty() {
"this template doesn't take picked LoRAs".to_string()
} else {
format!(
"this template takes at most {} picked LoRAs",
self.slots.len()
)
}));
}
if let Some(unknown) = loras
.keys()
.find(|id| !self.loras.iter().any(|l| &l.cfg.id == *id))
@@ -362,6 +472,26 @@ impl Template {
}
}
// Picked LoRAs fill the slots in order; the rest run at 0 (their
// exported lora_name stays, a file ComfyUI knows).
for (i, slot) in self.slots.iter().enumerate() {
let pick = picks.get(i).filter(|p| p.scale > 0.0);
for (nodes, file) in [
(&slot.high, pick.map(|p| p.high.as_str())),
(&slot.low, pick.map(|p| p.low.as_str())),
] {
for (id, exported) in nodes {
if let Some(file) = file {
set_input(&mut wf, id, "lora_name", json!(file));
}
let scale = pick.map_or(0.0, |p| p.scale);
for (input, value) in exported {
set_input(&mut wf, id, input, json!(value * scale));
}
}
}
}
for node in wf.values_mut() {
let Some(inputs) = node.get_mut("inputs").and_then(Value::as_object_mut) else {
continue;
@@ -435,6 +565,17 @@ mod tests {
/// The Animate template plus a Style pair titled `imageapi:lora:style`
/// (nodes 50 high, 51 low).
pub const ANIMATE_LORAS: &str = include_str!("../../tests/fixtures/enhance/animate_loras.json");
/// ANIMATE_LORAS plus two pickable slots chained after the switch:
/// 60/61 (slot 1 high/low), 62/63 (slot 2).
pub const ANIMATE_SLOTS: &str = include_str!("../../tests/fixtures/enhance/animate_slots.json");
fn pick(name: &str, scale: f64) -> SlotPick {
SlotPick {
high: format!("{name}_high.safetensors"),
low: format!("{name}_low.safetensors"),
scale,
}
}
fn lora(id: &str, default: bool) -> LoraConfig {
LoraConfig {
@@ -475,6 +616,7 @@ mod tests {
prompt_join: " ".into(),
output_megapixels: 1.0,
loras: Vec::new(),
catalog: Default::default(),
}
}
@@ -547,6 +689,7 @@ mod tests {
Some("more"),
None,
&BTreeMap::new(),
&[],
&mut seeds(),
)
.unwrap();
@@ -566,7 +709,7 @@ mod tests {
fn fill_sets_path_forces_approval_and_randomises_every_seed() {
let t = load(RESTORE, PromptMode::Append).unwrap();
let f = t
.fill("Z:\\p\\a.jpg", None, None, &none(), &mut seeds())
.fill("Z:\\p\\a.jpg", None, None, &none(), &[], &mut seeds())
.unwrap();
assert_eq!(input(&f.prompt, "37", "path"), "Z:\\p\\a.jpg");
assert_eq!(input(&f.prompt, "36", "require_approval"), true);
@@ -579,8 +722,8 @@ mod tests {
assert_ne!(a, 774744764437710);
// Two fills with a real RNG differ.
let mut rng = random_seed;
let x = t.fill("p", None, None, &none(), &mut rng).unwrap();
let y = t.fill("p", None, None, &none(), &mut rng).unwrap();
let x = t.fill("p", None, None, &none(), &[], &mut rng).unwrap();
let y = t.fill("p", None, None, &none(), &[], &mut rng).unwrap();
assert_ne!(input(&x.prompt, "5", "seed"), input(&y.prompt, "5", "seed"));
}
@@ -606,7 +749,9 @@ mod tests {
// SeedVR2VideoUpscaler rejects seeds above 2^32 - 1 at validation.
let t = load(RESTORE, PromptMode::Append).unwrap();
for _ in 0..200 {
let f = t.fill("p", None, None, &none(), &mut random_seed).unwrap();
let f = t
.fill("p", None, None, &none(), &[], &mut random_seed)
.unwrap();
assert!(input(&f.prompt, "5", "seed").as_u64().unwrap() <= u64::from(u32::MAX));
}
}
@@ -615,7 +760,7 @@ mod tests {
fn force_approval_even_if_the_export_had_it_off() {
let off = RESTORE.replace("\"require_approval\": true", "\"require_approval\": false");
let t = load(&off, PromptMode::Append).unwrap();
let f = t.fill("p", None, None, &none(), &mut seeds()).unwrap();
let f = t.fill("p", None, None, &none(), &[], &mut seeds()).unwrap();
assert_eq!(input(&f.prompt, "36", "require_approval"), true);
}
@@ -633,6 +778,7 @@ mod tests {
Some(" warmer "),
None,
&BTreeMap::new(),
&[],
&mut seeds(),
)
.unwrap();
@@ -641,10 +787,10 @@ mod tests {
"Increase the clarity\n\nAdditional instructions: warmer"
);
let f = t
.fill("p", Some(" "), None, &none(), &mut seeds())
.fill("p", Some(" "), None, &none(), &[], &mut seeds())
.unwrap();
assert_eq!(input(&f.prompt, "13", "value"), base);
let f = t.fill("p", None, None, &none(), &mut seeds()).unwrap();
let f = t.fill("p", None, None, &none(), &[], &mut seeds()).unwrap();
assert_eq!(input(&f.prompt, "13", "value"), base);
}
@@ -657,6 +803,7 @@ mod tests {
Some("Make it warmer."),
None,
&BTreeMap::new(),
&[],
&mut seeds(),
)
.unwrap();
@@ -667,10 +814,12 @@ mod tests {
#[test]
fn replace_mode_requires_and_uses_the_instruction() {
let t = load(EDIT, PromptMode::Replace).unwrap();
let err = t.fill("p", None, None, &none(), &mut seeds()).unwrap_err();
let err = t
.fill("p", None, None, &none(), &[], &mut seeds())
.unwrap_err();
assert!(err.0.contains("instruction"));
assert!(
t.fill("p", Some(" "), None, &none(), &mut seeds())
t.fill("p", Some(" "), None, &none(), &[], &mut seeds())
.is_err()
);
let f = t
@@ -679,6 +828,7 @@ mod tests {
Some("make it black and white"),
None,
&BTreeMap::new(),
&[],
&mut seeds(),
)
.unwrap();
@@ -692,13 +842,16 @@ mod tests {
// 14 links to 13; drop that link so the graph stays plausible.
v["14"]["inputs"].as_object_mut().unwrap().remove("prompt");
let t = Template::from_value(cfg("t", PromptMode::Append), v).unwrap();
assert!(t.fill("p", Some("x"), None, &none(), &mut seeds()).is_err());
assert!(t.fill("p", None, None, &none(), &mut seeds()).is_ok());
assert!(
t.fill("p", Some("x"), None, &none(), &[], &mut seeds())
.is_err()
);
assert!(t.fill("p", None, None, &none(), &[], &mut seeds()).is_ok());
let restore = load(RESTORE, PromptMode::Append).unwrap();
assert!(
restore
.fill("p", None, Some("r"), &none(), &mut seeds())
.fill("p", None, Some("r"), &none(), &[], &mut seeds())
.is_err()
);
}
@@ -712,6 +865,7 @@ mod tests {
Some("x"),
Some("Z:\\ref.jpg"),
&BTreeMap::new(),
&[],
&mut seeds(),
)
.unwrap();
@@ -724,7 +878,7 @@ mod tests {
fn no_reference_removes_the_node_and_every_link_to_it() {
let t = load(EDIT, PromptMode::Replace).unwrap();
let f = t
.fill("src", Some("x"), None, &none(), &mut seeds())
.fill("src", Some("x"), None, &none(), &[], &mut seeds())
.unwrap();
assert!(f.dropped_reference);
let wf = f.prompt.as_object().unwrap();
@@ -751,7 +905,7 @@ mod tests {
assert!(t.warnings.is_empty(), "{:?}", t.warnings);
// Default off: both the high and the low node go to 0.
let off = t.fill("p", None, None, &none(), &mut seeds()).unwrap();
let off = t.fill("p", None, None, &none(), &[], &mut seeds()).unwrap();
assert_eq!(off.loras, scales(&[("style", 0.0)]));
for id in ["50", "51"] {
assert_eq!(input(&off.prompt, id, "strength_model"), &json!(0.0));
@@ -767,6 +921,7 @@ mod tests {
None,
None,
&reqs(&[("style", LoraRequest::On(true))]),
&[],
&mut seeds(),
)
.unwrap();
@@ -791,6 +946,7 @@ mod tests {
None,
None,
&reqs(&[("style", LoraRequest::Scale(0.5))]),
&[],
&mut seeds(),
)
.unwrap();
@@ -805,6 +961,7 @@ mod tests {
None,
None,
&reqs(&[("style", LoraRequest::Scale(9.0))]),
&[],
&mut seeds(),
)
.unwrap();
@@ -815,6 +972,7 @@ mod tests {
None,
None,
&reqs(&[("style", LoraRequest::Scale(0.0))]),
&[],
&mut seeds(),
)
.unwrap();
@@ -826,7 +984,7 @@ mod tests {
let mut l = lora("style", true);
l.strength = 0.7;
let t = load_with_loras(ANIMATE_LORAS, vec![l]).unwrap();
let f = t.fill("p", None, None, &none(), &mut seeds()).unwrap();
let f = t.fill("p", None, None, &none(), &[], &mut seeds()).unwrap();
assert_eq!(f.loras, scales(&[("style", 0.7)]));
assert_eq!(input(&f.prompt, "50", "strength_model"), &json!(0.7));
@@ -836,6 +994,7 @@ mod tests {
None,
None,
&reqs(&[("nope", LoraRequest::On(true))]),
&[],
&mut seeds(),
)
.unwrap_err();
@@ -850,7 +1009,7 @@ mod tests {
let mut c = cfg("animate", PromptMode::Append);
c.loras = vec![lora("style", false)];
let t = Template::from_value(c, v).unwrap();
let off = t.fill("p", None, None, &none(), &mut seeds()).unwrap();
let off = t.fill("p", None, None, &none(), &[], &mut seeds()).unwrap();
assert_eq!(input(&off.prompt, "50", "strength_clip"), &json!(0.0));
let half = t
.fill(
@@ -858,6 +1017,7 @@ mod tests {
None,
None,
&reqs(&[("style", LoraRequest::Scale(0.5))]),
&[],
&mut seeds(),
)
.unwrap();
@@ -874,7 +1034,7 @@ mod tests {
"{:?}",
t.warnings
);
let f = t.fill("p", None, None, &none(), &mut seeds()).unwrap();
let f = t.fill("p", None, None, &none(), &[], &mut seeds()).unwrap();
assert!(f.loras.is_empty());
assert_eq!(input(&f.prompt, "50", "strength_model"), &json!(1.0));
assert!(
@@ -883,7 +1043,8 @@ mod tests {
None,
None,
&reqs(&[("style", LoraRequest::On(false))]),
&mut seeds()
&[],
&mut seeds(),
)
.is_err()
);
@@ -917,4 +1078,106 @@ mod tests {
.to_string();
assert!(err.contains("strength_model"), "{err}");
}
#[test]
fn slots_are_found_and_fixed_loras_listed() {
let t = load(ANIMATE_SLOTS, PromptMode::Append).unwrap();
assert_eq!(t.slot_count(), 2);
let mut fixed: Vec<_> = t.fixed_loras().iter().cloned().collect();
fixed.sort();
assert_eq!(
fixed,
[
"example_style_high.safetensors",
"example_style_low.safetensors",
"wan2.2_i2v_lightx2v_4steps_lora_v1_high_noise.safetensors",
"wan2.2_i2v_lightx2v_4steps_lora_v1_low_noise.safetensors",
]
);
assert_eq!(
load(ANIMATE_LORAS, PromptMode::Append)
.unwrap()
.slot_count(),
0
);
}
#[test]
fn picks_fill_slots_in_order_and_unused_slots_run_at_zero() {
let t = load(ANIMATE_SLOTS, PromptMode::Append).unwrap();
let f = t
.fill(
"p",
None,
None,
&none(),
&[pick("motion", 0.5)],
&mut seeds(),
)
.unwrap();
assert_eq!(
input(&f.prompt, "60", "lora_name"),
&json!("motion_high.safetensors")
);
assert_eq!(
input(&f.prompt, "61", "lora_name"),
&json!("motion_low.safetensors")
);
assert_eq!(input(&f.prompt, "60", "strength_model"), &json!(0.5));
assert_eq!(input(&f.prompt, "61", "strength_model"), &json!(0.5));
// Slot 2 unused: its exported file stays, at strength 0.
assert_eq!(
input(&f.prompt, "62", "lora_name"),
&json!("example_slot_high.safetensors")
);
assert_eq!(input(&f.prompt, "62", "strength_model"), &json!(0.0));
assert_eq!(input(&f.prompt, "63", "strength_model"), &json!(0.0));
// The chain isn't rewired.
assert_eq!(input(&f.prompt, "14", "model"), &json!(["62", 0]));
// No picks at all: every slot off.
let f = t.fill("p", None, None, &none(), &[], &mut seeds()).unwrap();
assert_eq!(input(&f.prompt, "60", "strength_model"), &json!(0.0));
let err = t
.fill(
"p",
None,
None,
&none(),
&[pick("a", 1.0), pick("b", 1.0), pick("c", 1.0)],
&mut seeds(),
)
.unwrap_err();
assert!(err.0.contains("at most 2"), "{err}");
let plain = load(ANIMATE_LORAS, PromptMode::Append).unwrap();
assert!(
plain
.fill("p", None, None, &none(), &[pick("a", 1.0)], &mut seeds())
.is_err()
);
}
#[test]
fn malformed_slots_are_rejected() {
let only_high = ANIMATE_SLOTS.replace("imageapi:lora-slot:2:low", "something else");
let err = load(&only_high, PromptMode::Append)
.unwrap_err()
.to_string();
assert!(err.contains("slot 2 needs both"), "{err}");
let gap = ANIMATE_SLOTS
.replace("imageapi:lora-slot:2:high", "imageapi:lora-slot:3:high")
.replace("imageapi:lora-slot:2:low", "imageapi:lora-slot:3:low");
let err = load(&gap, PromptMode::Append).unwrap_err().to_string();
assert!(err.contains("without gaps"), "{err}");
let role = ANIMATE_SLOTS.replace("imageapi:lora-slot:1:high", "imageapi:lora-slot:1:mid");
assert!(
load(&role, PromptMode::Append)
.unwrap_err()
.to_string()
.contains("high or low")
);
let no_n = ANIMATE_SLOTS.replace("imageapi:lora-slot:1:high", "imageapi:lora-slot:x:high");
assert!(load(&no_n, PromptMode::Append).is_err());
}
}