From 0c5156ecc8dfcefd6633d391cd43a337aabd609b Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 8 Jun 2026 07:25:19 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: crumb/bloom-560m-RLHF-SD2-prompter-aesthetic Source: Original Platform --- .gitattributes | 35 ++++++++++++++++++++ README.md | 72 +++++++++++++++++++++++++++++++++++++++++ config.json | 33 +++++++++++++++++++ model.safetensors | 3 ++ pytorch_model.bin | 3 ++ special_tokens_map.json | 6 ++++ tokenizer.json | 3 ++ tokenizer_config.json | 11 +++++++ 8 files changed, 166 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 model.safetensors create mode 100644 pytorch_model.bin create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9465f3d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..26243e5 --- /dev/null +++ b/README.md @@ -0,0 +1,72 @@ +--- +license: bigscience-bloom-rail-1.0 +tags: +- stable-diffusion +- diffusion +model-index: +- name: bloom-560m-RLHF-SD2-prompter + results: [] + +datasets: + - Gustavosta/Stable-Diffusion-Prompts + +widget: +- text: "Prompt: " + +inference: + parameters: + eos_token_id: 2 + max_length: 128 + do_sample: true +--- + +# The RAT (RLHF-Aesthetic Tuned model for prompt synthesis) + +**COLAB DEMO INCLUDING STABLE DIFFUSION: https://colab.research.google.com/github/aicrumb/doohickey/blob/main/rlhf_prompt_tuner.ipynb** + +This is a further finetuned version of [crumb/bloom-560m-RLHF-SD2-prompter](https://hf.co/crumb/bloom-560m-RLHF-SD2-prompter) to optimize for aesthetic score with models from https://github.com/crowsonkb/simulacra-aesthetic-models instead of me hand scoring each image + +donate so i can do this on real hardware : https://github.com/aicrumb/aicrumb/blob/main/README.md + +trained at bs=32, lr=0.0001, only tuning biases and layernorm weights + +## Example usage + +```python +# Install libraries needed to run the models +!pip install transformers diffusers accelerate -qq + +# Import the libraries +from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler +from transformers import pipeline +import torch + +# This is the model that the transformer was finetuned to generate prompts for +model_id = "stabilityai/stable-diffusion-2-base" + +# Use the Euler scheduler here +scheduler = EulerDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") +pipe = StableDiffusionPipeline.from_pretrained(model_id, scheduler=scheduler, revision="fp16", torch_dtype=torch.float16) +pipe = pipe.to("cuda") + +# Load the transformer model +prompt_pipe = pipeline("text-generation", model="crumb/bloom-560m-RLHF-SD2-prompter-aesthetic") +prompt = "cool landscape" + +# Auto-complete prompt +prompt = "Prompt: " + prompt + "," +extended_prompt = prompt_pipe(prompt, do_sample=True, max_length=42)[0]['generated_text'] +extended_prompt = extended_prompt[10:] +print("Prompt is now: ", extended_prompt) + +# Generate image +image = pipe(extended_prompt).images[0] + +image.save("output.png") +image +``` + +## Limitations +Aesthetic scoring models have been shown to have very large biases, and one I noticed is it really likes images of women no matter the actual quality, so those were optimized for more than other things. + +Also it fell into the trap of rlhf models, it gets kinda same-ey, so if you don't like the general "stable diffusion, trending on artstation" look this might not be for you. \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..0a296ca --- /dev/null +++ b/config.json @@ -0,0 +1,33 @@ +{ + "_name_or_path": "crumb/bloom-560m-RLHF-SD2-prompter-aesthetic", + "apply_residual_connection_post_layernorm": false, + "architectures": [ + "BloomForCausalLM" + ], + "attention_dropout": 0.0, + "attention_softmax_in_fp32": true, + "bias_dropout_fusion": true, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_dropout": 0.0, + "hidden_size": 1024, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "masked_softmax_fusion": true, + "model_type": "bloom", + "n_head": 16, + "n_inner": null, + "n_layer": 24, + "offset_alibi": 100, + "pad_token_id": 3, + "pretraining_tp": 1, + "seq_length": 2048, + "skip_bias_add": true, + "skip_bias_add_qkv": false, + "slow_but_exact": false, + "torch_dtype": "float16", + "transformers_version": "4.24.0", + "unk_token_id": 0, + "use_cache": false, + "vocab_size": 250880 +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..e99986f --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fd322f8f5be8b7449601b38b7aba6fc4ad2c7c75b0b6a803249e817a3216a02 +size 1118462962 diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..a97f83a --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97ec104e76ad311e34a3098abf85bb466a56583a6ca08ca19f5106fe8d312fd2 +size 1118526007 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..fdafe48 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,6 @@ +{ + "bos_token": "", + "eos_token": "", + "pad_token": "", + "unk_token": "" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..a4fa803 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5 +size 14500443 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..9db6289 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,11 @@ +{ + "add_prefix_space": false, + "bos_token": "", + "eos_token": "", + "name_or_path": "crumb/bloom-560m-RLHF-SD2-prompter", + "pad_token": "", + "padding_side": "left", + "special_tokens_map_file": null, + "tokenizer_class": "BloomTokenizer", + "unk_token": "" +}