commit d86266b02d36e3ce0e1521140c5bfdaf5c01cdad Author: ModelHub XC Date: Sun Oct 25 02:14:17 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: dejanseo/reverse-prompter Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5b0aebf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +assets/train-loss.png filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..645e9e2 --- /dev/null +++ b/README.md @@ -0,0 +1,117 @@ +--- +license: other +license_name: link-attribution +license_link: https://dejanmarketing.com/link-attribution/ +library_name: transformers +base_model: google/gemma-3-270m +tags: + - reverse-prompting + - prompt-reconstruction + - gemma + - text-generation +pipeline_tag: text-generation +--- + +# Reverse Prompter + +A fine-tuned [google/gemma-3-270m](https://huggingface.co/google/gemma-3-270m) model that reconstructs the most likely prompt from an AI assistant's response. + +Given an AI-generated text, the model generates candidate prompts that could have produced it. + +## How It Works + +The model was trained on prompt-response pairs formatted as: + +``` +{response}\n###\n{prompt} +``` + +At inference time, you provide the response followed by the `\n###\n` separator, and the model generates the reconstructed prompt. + +## Usage + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model = AutoModelForCausalLM.from_pretrained("dejanseo/reverse-prompter", torch_dtype="bfloat16").cuda().eval() +tokenizer = AutoTokenizer.from_pretrained("dejanseo/reverse-prompter") + +response_text = "Your AI-generated text here" +prompt = response_text.strip() + "\n###\n" + +inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False).to(model.device) +outputs = model.generate(**inputs, max_new_tokens=256, penalty_alpha=0.3, top_k=4) + +generated = outputs[0][inputs["input_ids"].shape[-1]:] +reconstructed_prompt = tokenizer.decode(generated, skip_special_tokens=True).strip() +print(reconstructed_prompt) +``` + +For best results, run generation across multiple contrastive search configurations and rank outputs by perplexity. See the companion Streamlit app for a full implementation. + +## Training Data + +The training dataset was generated synthetically using Gemini 2.5 Flash via Vertex AI in a three-stage pipeline: + +### 1. Prompt Generation + +100,000 diverse prompts were generated across five categories (20 each per batch): + +- Mid-tail, search query style (single or multi-faceted) +- Long-tail, search query style (multi-faceted) +- Simple, prompt-like (single-faceted) +- Typical, prompt-like (single or multi-faceted) +- Detailed, prompt-like (multi-faceted) + +Generation was parallelized with 100 concurrent API calls in batches of 100 prompts, with results stored in SQLite. + +### 2. Response Generation + +Each prompt was sent back to Gemini 2.5 Flash (with thinking disabled) to produce a corresponding AI assistant response. This was also parallelized at 100 concurrent calls. + +### 3. Tokenization + +Prompt-response pairs were formatted as `{response}\n###\n{prompt}` and tokenized using the Gemma 3 tokenizer. Labels were masked (`-100`) over the response and separator tokens so the model only learns to predict the prompt portion. Tokenization was done in batches of 5,000 and concatenated into the final dataset. + +## Training Details + +| Parameter | Value | +|---|---| +| Base model | google/gemma-3-270m | +| Method | Full fine-tune | +| Precision | bfloat16 | +| Epochs | 1 | +| Batch size | 2 | +| Gradient accumulation | 8 (effective batch size 16) | +| Learning rate | 5e-5 | +| Warmup steps | 100 | +| Max sequence length | 2048 | +| Optimizer | AdamW (torch fused) | +| Gradient checkpointing | Enabled | +| Training time | 4h 14m | +| GPU | NVIDIA GeForce RTX 4090 (24 GB) | +| CPU | AMD Ryzen 9 7950X3D 16-Core | +| RAM | 128 GB | + +### Training Loss + +![Training Loss](assets/train-loss.png) + +## Inference Strategy + +The companion app uses contrastive search with a sweep over configurations: + +- `top_k`: [2, 4, 6, 15] +- `penalty_alpha`: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6] + +This produces up to 24 candidate prompts per input. Candidates are deduplicated and ranked by perplexity (lower is better). Token-level probabilities provide a confidence signal for each word in the reconstruction. + +## Limitations + +- Prompt reconstruction is inherently probabilistic. The model returns plausible prompts, not necessarily the exact original. +- Performance is best on responses typical of AI assistants. Non-standard or very short inputs may produce lower-quality reconstructions. +- The model inherits the capabilities and limitations of the gemma-3-270m base model. + +## Author + +[Dejan AI](https://dejan.ai/) diff --git a/assets/train-270m-ft.py b/assets/train-270m-ft.py new file mode 100644 index 0000000..309276e --- /dev/null +++ b/assets/train-270m-ft.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Full fine-tune gemma-3-270m to reconstruct prompts from model outputs.""" + +from datasets import load_from_disk +from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments, DataCollatorForSeq2Seq + +# --- Configuration --- +MODEL_NAME = "google/gemma-3-270m" +DATASET_PATH = "tokenized-dataset-plain" +OUTPUT_DIR = "checkpoints-270m-ft" +MAX_SEQ_LENGTH = 2048 + +# --- Training --- +BATCH_SIZE = 2 +GRAD_ACCUM = 8 +LEARNING_RATE = 5e-5 +NUM_EPOCHS = 1 +WARMUP_STEPS = 100 +LOGGING_STEPS = 1 +SAVE_STEPS = 100 + + +def main(): + print("Loading pre-tokenized dataset...") + dataset = load_from_disk(DATASET_PATH) + print(f"Training examples: {len(dataset)}") + + print(f"Loading model: {MODEL_NAME}") + model = AutoModelForCausalLM.from_pretrained( + MODEL_NAME, + torch_dtype="bfloat16", + device_map="auto", + ) + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + + trainer = Trainer( + model=model, + processing_class=tokenizer, + train_dataset=dataset, + data_collator=DataCollatorForSeq2Seq(tokenizer, padding=True), + args=TrainingArguments( + output_dir=OUTPUT_DIR, + per_device_train_batch_size=BATCH_SIZE, + gradient_accumulation_steps=GRAD_ACCUM, + learning_rate=LEARNING_RATE, + num_train_epochs=NUM_EPOCHS, + warmup_steps=WARMUP_STEPS, + logging_steps=LOGGING_STEPS, + save_steps=SAVE_STEPS, + bf16=True, + seed=42, + report_to="wandb", + logging_strategy="steps", + gradient_checkpointing=True, + ), + ) + + print("Training...") + trainer.train() + + print("Saving model...") + trainer.save_model(f"{OUTPUT_DIR}/final") + tokenizer.save_pretrained(f"{OUTPUT_DIR}/final") + print("Done.") + + +if __name__ == "__main__": + main() diff --git a/assets/train-loss.png b/assets/train-loss.png new file mode 100644 index 0000000..c371719 --- /dev/null +++ b/assets/train-loss.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22e27d2504371f7a5c14d1cf816fe3a84e2fb548308a517496d36db303f44165 +size 116660 diff --git a/config.json b/config.json new file mode 100644 index 0000000..282ec64 --- /dev/null +++ b/config.json @@ -0,0 +1,62 @@ +{ + "_sliding_window_pattern": 6, + "architectures": [ + "Gemma3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 640, + "initializer_range": 0.02, + "intermediate_size": 2048, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "model_type": "gemma3_text", + "num_attention_heads": 4, + "num_hidden_layers": 18, + "num_key_value_heads": 1, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 512, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262144 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..8e8e8eb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.2.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..4604d47 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29fcb7e33e2d86bdbd993d6225405391c76aca4c8fbda11c5b346fe5fac16a82 +size 536223056 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..899af07 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497 +size 33384443 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..7202584 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,24 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "mask_token": "", + "model_max_length": 1000000000000000019884624838656, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..3fe2632 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40d1bee8f175e5cd936a9ba3986b2c52791fe8d8152bcf10393ebec85ea1cff5 +size 5201