mingyi456/NetaYume-Lumina-Image-2.0-DF11

For more information (including how to compress models yourself), check out https://huggingface.co/DFloat11 and https://github.com/LeanModels/DFloat11

Feel free to request for other models for compression as well (for either the diffusers library, ComfyUI, or any other model), although models that use architectures which are unfamiliar to me might be more difficult.

How to Use

diffusers

import torch
from diffusers import from diffusers import Lumina2Transformer2DModel, Lumina2Pipeline
# from transformers.modeling_utils import no_init_weights # for transformers<5.0.0
from transformers.initialization import no_init_weights # for transformers>=5.0.0

pattern_dict = {
    r"noise_refiner\.\d+": (
        "attn.to_q",
        "attn.to_k",
        "attn.to_v",
        "attn.to_out.0",
        "feed_forward.linear_1",
        "feed_forward.linear_2",
        "feed_forward.linear_3",
        "norm1.linear"
    ),
    r"context_refiner\.\d+": (
        "attn.to_q",
        "attn.to_k",
        "attn.to_v",
        "attn.to_out.0",
        "feed_forward.linear_1",
        "feed_forward.linear_2",
        "feed_forward.linear_3",
    ),
    r"layers\.\d+": (
        "attn.to_q",
        "attn.to_k",
        "attn.to_v",
        "attn.to_out.0",
        "feed_forward.linear_1",
        "feed_forward.linear_2",
        "feed_forward.linear_3",
        "norm1.linear"
    ),
    r"time_caption_embed\.caption_embedder": (
        "1",
    )
}

with no_init_weights():
    transformer = Lumina2Transformer2DModel.from_config(
        Lumina2Transformer2DModel.load_config(
            "duongve/NetaYume-Lumina-Image-2.0-Diffusers-v40", subfolder="transformer"
        ),
        torch_dtype=torch.bfloat16
    ).to(torch.bfloat16)
# Make sure to download the file first, and edit the filepath accordingly
DFloat11Model.from_single_file(
    "NetaYume-Lumina-Image-2.0-DF11/NetaYumev4_unet-DF11.safetensors",
    device="cpu",
    bfloat16_model=transformer,
    pattern_dict=pattern_dict,
)
pipe = Lumina2Pipeline.from_pretrained(
    "duongve/NetaYume-Lumina-Image-2.0-Diffusers-v40",
    transformer=transformer, 
    torch_dtype=torch.bfloat16
)
pipe.enable_model_cpu_offload() 
prompt = "kita ikuyo (Bocchi the Rock!), 1girl, anime style, vibrant colors, red hair, medium hair with one side up, green eyes, bangs, hair between eyes, school uniform (white shirt, grey serafuku sailor collar, red neckerchief, pleated skirt), sitting upper body close-up, holding bouquet with white lily & pink flowers, indoors with depth of field, cherry blossom-like light particles, soft sunlight backlighting, bloom, chromatic aberration & lens flare abuse, light smile, closed mouth, one side hair up, transparent blurry foreground, warm cozy atmosphere, masterpiece, best quality"
image = pipe(
    prompt,
    height=1536,
    width=1024,
    guidance_scale=4.0,
    num_inference_steps=50,
    cfg_trunc_ratio=6,
    cfg_normalization=False, #Important
    generator=torch.Generator("cuda").manual_seed(0),
    system_prompt="You are an assistant designed to generate anime images based on textual prompts.",
).images[0]
image.save('image netayume.png')

ComfyUI

Refer to this model instead.

Compression details

This is the pattern_dict for compression:

pattern_dict = {
    r"noise_refiner\.\d+": (
        "attn.to_q",
        "attn.to_k",
        "attn.to_v",
        "attn.to_out.0",
        "feed_forward.linear_1",
        "feed_forward.linear_2",
        "feed_forward.linear_3",
        "norm1.linear"
    ),
    r"context_refiner\.\d+": (
        "attn.to_q",
        "attn.to_k",
        "attn.to_v",
        "attn.to_out.0",
        "feed_forward.linear_1",
        "feed_forward.linear_2",
        "feed_forward.linear_3",
    ),
    r"layers\.\d+": (
        "attn.to_q",
        "attn.to_k",
        "attn.to_v",
        "attn.to_out.0",
        "feed_forward.linear_1",
        "feed_forward.linear_2",
        "feed_forward.linear_3",
        "norm1.linear"
    ),
    r"time_caption_embed\.caption_embedder": (
        "1",
    )
}