deepgenteam

/

DeepGen-1.0-diffusers

like 3

Follow

deepgen 62

This patch is aims to fix

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[3], line 1
----> 1 result = pipe(
      2     prompt="a racoon holding a shiny red apple over its head",
      3     height=512, width=512,
      4     num_inference_steps=50,
      5     guidance_scale=4.0,
      6     seed=42,
      7 )

File ~/Models/Playground/venv/lib/python3.12/site-packages/torch/utils/_contextlib.py:124, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    120 @functools.wraps(func)
    121 def decorate_context(*args, **kwargs):
    122     # pyrefly: ignore [bad-context-manager]
    123     with ctx_factory():
--> 124         return func(*args, **kwargs)

File ~/.cache/huggingface/modules/diffusers_modules/local/deepgen_pipeline.py:1362, in DeepGenPipeline.__call__(self, prompt, image, negative_prompt, height, width, num_inference_steps, guidance_scale, seed, num_images_per_prompt)
   1354     self._offload_to(self.transformer, gpu)
   1356 pipeline = _SD3Pipeline(
   1357     transformer=self.transformer, scheduler=self.scheduler,
   1358     vae=self.vae, text_encoder=None, tokenizer=None,
   1359     text_encoder_2=None, tokenizer_2=None,
   1360     text_encoder_3=None, tokenizer_3=None)
-> 1362 samples = pipeline(
   1363     height=height, width=width,
   1364     guidance_scale=guidance_scale,
   1365     num_inference_steps=num_inference_steps,
   1366     prompt_embeds=seq_out[:b],
   1367     pooled_prompt_embeds=pooled_out[:b],
   1368     negative_prompt_embeds=seq_out[b:],
   1369     negative_pooled_prompt_embeds=pooled_out[b:],
   1370     generator=generator,
   1371     output_type='latent',
   1372     cond_latents=cond_latents,
   1373 ).images.to(self.transformer.dtype)
   1375 if offload:
   1376     self._offload_to(self.transformer, "cpu")

File ~/Models/Playground/venv/lib/python3.12/site-packages/torch/utils/_contextlib.py:124, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    120 @functools.wraps(func)
    121 def decorate_context(*args, **kwargs):
    122     # pyrefly: ignore [bad-context-manager]
    123     with ctx_factory():
--> 124         return func(*args, **kwargs)

File ~/.cache/huggingface/modules/diffusers_modules/local/deepgen_pipeline.py:864, in _SD3Pipeline.__call__(self, prompt, prompt_2, prompt_3, height, width, num_inference_steps, sigmas, guidance_scale, negative_prompt, negative_prompt_2, negative_prompt_3, num_images_per_prompt, generator, latents, cond_latents, prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds, output_type, return_dict, joint_attention_kwargs, callback_on_step_end, callback_on_step_end_tensor_inputs, max_sequence_length, mu, **kwargs)
    862 latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
    863 timestep = t.expand(latent_model_input.shape[0])
--> 864 noise_pred = self.transformer(
    865     hidden_states=latent_model_input, cond_hidden_states=cond_latents,
    866     timestep=timestep, encoder_hidden_states=prompt_embeds,
    867     pooled_projections=pooled_prompt_embeds,
    868     joint_attention_kwargs=self.joint_attention_kwargs,
    869     return_dict=False)[0]
    871 if self.do_classifier_free_guidance:
    872     noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)

File ~/Models/Playground/venv/lib/python3.12/site-packages/torch/nn/modules/module.py:1776, in Module._wrapped_call_impl(self, *args, **kwargs)
   1774     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1775 else:
-> 1776     return self._call_impl(*args, **kwargs)

File ~/Models/Playground/venv/lib/python3.12/site-packages/torch/nn/modules/module.py:1787, in Module._call_impl(self, *args, **kwargs)
   1782 # If we don't have any hooks, we want to skip the rest of the logic in
   1783 # this function, and just call forward.
   1784 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1785         or _global_backward_pre_hooks or _global_backward_hooks
   1786         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1787     return forward_call(*args, **kwargs)
   1789 result = None
   1790 called_always_called_hooks = set()

File ~/Models/Playground/venv/lib/python3.12/site-packages/accelerate/hooks.py:175, in add_hook_to_module.<locals>.new_forward(module, *args, **kwargs)
    173         output = module._old_forward(*args, **kwargs)
    174 else:
--> 175     output = module._old_forward(*args, **kwargs)
    176 return module._hf_hook.post_forward(module, output)

TypeError: SD3Transformer2DModel.forward() got an unexpected keyword argument 'cond_hidden_states'

diffuser will automatic load orginal SD3Transformer2DModel which doesnt support cond_hidden_states, original code tries to "hack" the class of the transformers bnut it may failed to replace the forward method or properly initialise the custom class structure