初始化项目，由ModelHub XC社区提供模型
Model: strangervisionhf/dots.ocr-base-fix Source: Original Platform
This commit is contained in:
ModelHub XC
300
README.md
Normal file
300
README.md
Normal file
@@ -0,0 +1,300 @@
|
||||
---
|
||||
license: mit
|
||||
library_name: transformers
|
||||
language:
|
||||
- en
|
||||
pipeline_tag: image-text-to-text
|
||||
tags:
|
||||
- text-generation-inference
|
||||
- OCR
|
||||
- VLM
|
||||
- Markdown
|
||||
- pytorch
|
||||
new_version: prithivMLmods/Dots.OCR-Latest-BF16
|
||||
---
|
||||
|
||||
> [!warning]
|
||||
This version is experimental. Please refer to the newer versions pinned above to avoid any complexities.👆👆👆
|
||||
|
||||
|
||||
> [!IMPORTANT]
|
||||
> This is a copy of the model weights from the [https://huggingface.co/rednote-hilab/dots.ocr](https://huggingface.co/rednote-hilab/dots.ocr) model. These weights cannot be used for other purposes. If you wish to do so, please visit the original model page.
|
||||
|
||||
Previously, inference with the model [[https://huggingface.co/rednote-hilab/dots.ocr](https://huggingface.co/rednote-hilab/dots.ocr)] would fail with the following error:
|
||||
**Error loading dots-ocr model: Received a NoneType for argument 'video_processor', but a BaseVideoProcessor was expected.** in the latest Transformers versions.
|
||||
|
||||
This page, which includes the model weights and corrected configuration, fixed the issue and allowed Transformers inference to run smoothly.
|
||||
|
||||
> [!note]
|
||||
Last updated: 5:00 AM (IST), October 25, 2025.
|
||||
|
||||
> [!note]
|
||||
A PR to fix the issue has been raised on the original model page **[PR:38]**: [huggingface.co/rednote-hilab/dots.ocr/discussions/38](https://huggingface.co/rednote-hilab/dots.ocr/discussions/38)
|
||||
|
||||
> [!note]
|
||||
The latest transformers version used as of the above date is `transformers==4.57.1` and the torch version `2.8.0+cu126`
|
||||
|
||||
## Quick Start with Transformers
|
||||
|
||||
> #### Install the required packages
|
||||
|
||||
```py
|
||||
!pip install transformers torch torchvision gradio hf_xet \
|
||||
huggingface_hub pillow accelerate peft \
|
||||
matplotlib requests einops av sentencepiece\
|
||||
transformers-stream-generator
|
||||
```
|
||||
|
||||
```py
|
||||
flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.3/flash_attn-2.7.3+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
|
||||
```
|
||||
- FlashAttention requires L4 or higher GPUs [This includes GPUs like the A100, RTX 3090, RTX 4090, H100, etc...].
|
||||
|
||||
> ### notebook login
|
||||
|
||||
```py
|
||||
from huggingface_hub import notebook_login, HfApi
|
||||
notebook_login()
|
||||
```
|
||||
|
||||
> ### Run [app.py]
|
||||
|
||||
```py
|
||||
import os
|
||||
import sys
|
||||
from threading import Thread
|
||||
from typing import Iterable
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
import gradio as gr
|
||||
import spaces
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import (
|
||||
AutoModelForCausalLM,
|
||||
AutoProcessor,
|
||||
TextIteratorStreamer,
|
||||
)
|
||||
|
||||
from gradio.themes import Soft
|
||||
from gradio.themes.utils import colors, fonts, sizes
|
||||
|
||||
# --- Theme and CSS Setup ---
|
||||
colors.steel_blue = colors.Color(
|
||||
name="steel_blue",
|
||||
c50="#EBF3F8",
|
||||
c100="#D3E5F0",
|
||||
c200="#A8CCE1",
|
||||
c300="#7DB3D2",
|
||||
c400="#529AC3",
|
||||
c500="#4682B4",
|
||||
c600="#3E72A0",
|
||||
c700="#36638C",
|
||||
c800="#2E5378",
|
||||
c900="#264364",
|
||||
c950="#1E3450",
|
||||
)
|
||||
|
||||
class SteelBlueTheme(Soft):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
primary_hue: colors.Color | str = colors.gray,
|
||||
secondary_hue: colors.Color | str = colors.steel_blue,
|
||||
neutral_hue: colors.Color | str = colors.slate,
|
||||
text_size: sizes.Size | str = sizes.text_lg,
|
||||
font: fonts.Font | str | Iterable[fonts.Font | str] = (
|
||||
fonts.GoogleFont("Outfit"), "Arial", "sans-serif",
|
||||
),
|
||||
font_mono: fonts.Font | str | Iterable[fonts.Font | str] = (
|
||||
fonts.GoogleFont("IBM Plex Mono"), "ui-monospace", "monospace",
|
||||
),
|
||||
):
|
||||
super().__init__(
|
||||
primary_hue=primary_hue,
|
||||
secondary_hue=secondary_hue,
|
||||
neutral_hue=neutral_hue,
|
||||
text_size=text_size,
|
||||
font=font,
|
||||
font_mono=font_mono,
|
||||
)
|
||||
super().set(
|
||||
background_fill_primary="*primary_50",
|
||||
background_fill_primary_dark="*primary_900",
|
||||
body_background_fill="linear-gradient(135deg, *primary_200, *primary_100)",
|
||||
body_background_fill_dark="linear-gradient(135deg, *primary_900, *primary_800)",
|
||||
button_primary_text_color="white",
|
||||
button_primary_text_color_hover="white",
|
||||
button_primary_background_fill="linear-gradient(90deg, *secondary_500, *secondary_600)",
|
||||
button_primary_background_fill_hover="linear-gradient(90deg, *secondary_600, *secondary_700)",
|
||||
button_primary_background_fill_dark="linear-gradient(90deg, *secondary_600, *secondary_700)",
|
||||
button_primary_background_fill_hover_dark="linear-gradient(90deg, *secondary_500, *secondary_600)",
|
||||
slider_color="*secondary_500",
|
||||
slider_color_dark="*secondary_600",
|
||||
block_title_text_weight="600",
|
||||
block_border_width="3px",
|
||||
block_shadow="*shadow_drop_lg",
|
||||
button_primary_shadow="*shadow_drop_lg",
|
||||
button_large_padding="11px",
|
||||
color_accent_soft="*primary_100",
|
||||
block_label_background_fill="*primary_200",
|
||||
)
|
||||
|
||||
steel_blue_theme = SteelBlueTheme()
|
||||
|
||||
css = """
|
||||
#main-title h1 {
|
||||
font-size: 2.3em !important;
|
||||
}
|
||||
#output-title h2 {
|
||||
font-size: 2.1em !important;
|
||||
}
|
||||
"""
|
||||
|
||||
MAX_MAX_NEW_TOKENS = 4096
|
||||
DEFAULT_MAX_NEW_TOKENS = 2048
|
||||
MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
|
||||
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# Load Dots.OCR from the local, patched directory
|
||||
MODEL_PATH_D = "strangervisionhf/dots.ocr-base-fix"
|
||||
processor = AutoProcessor.from_pretrained(MODEL_PATH_D, trust_remote_code=True)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_PATH_D,
|
||||
attn_implementation="flash_attention_2",
|
||||
torch_dtype=torch.bfloat16,
|
||||
device_map="auto",
|
||||
trust_remote_code=True
|
||||
).eval()
|
||||
|
||||
|
||||
# --- Generation Function ---
|
||||
@spaces.GPU
|
||||
def generate_image(text: str, image: Image.Image,
|
||||
max_new_tokens: int = 1024,
|
||||
temperature: float = 0.6,
|
||||
top_p: float = 0.9,
|
||||
top_k: int = 50,
|
||||
repetition_penalty: float = 1.2):
|
||||
"""Generate responses for image input using the Dots.OCR model."""
|
||||
if image is None:
|
||||
yield "Please upload an image.", "Please upload an image."
|
||||
return
|
||||
|
||||
images = [image.convert("RGB")]
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image"}] + [{"type": "text", "text": text}]
|
||||
}
|
||||
]
|
||||
|
||||
prompt = processor.apply_chat_template(messages, add_generation_prompt=True)
|
||||
inputs = processor(text=prompt, images=images, return_tensors="pt").to(device)
|
||||
|
||||
streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)
|
||||
generation_kwargs = {
|
||||
**inputs,
|
||||
"streamer": streamer,
|
||||
"max_new_tokens": max_new_tokens,
|
||||
"temperature": temperature,
|
||||
"top_p": top_p,
|
||||
"top_k": top_k,
|
||||
"repetition_penalty": repetition_penalty,
|
||||
"do_sample": True
|
||||
}
|
||||
thread = Thread(target=model.generate, kwargs=generation_kwargs)
|
||||
thread.start()
|
||||
|
||||
buffer = ""
|
||||
for new_text in streamer:
|
||||
buffer += new_text.replace("<|im_end|>", "")
|
||||
yield buffer, buffer
|
||||
|
||||
with gr.Blocks(css=css, theme=steel_blue_theme) as demo:
|
||||
gr.Markdown("# **dots.ocr-base-fix**", elem_id="main-title")
|
||||
gr.Markdown("Powered by `Dots.OCR`")
|
||||
with gr.Row():
|
||||
with gr.Column(scale=2):
|
||||
image_query = gr.Textbox(label="Query Input", placeholder="Enter your query here...")
|
||||
image_upload = gr.Image(type="pil", label="Upload Image", height=320)
|
||||
image_submit = gr.Button("Submit", variant="primary")
|
||||
|
||||
with gr.Accordion("Advanced options", open=False):
|
||||
max_new_tokens = gr.Slider(label="Max new tokens", minimum=1, maximum=MAX_MAX_NEW_TOKENS, step=1, value=DEFAULT_MAX_NEW_TOKENS)
|
||||
temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=4.0, step=0.1, value=0.6)
|
||||
top_p = gr.Slider(label="Top-p (nucleus sampling)", minimum=0.05, maximum=1.0, step=0.05, value=0.9)
|
||||
top_k = gr.Slider(label="Top-k", minimum=1, maximum=1000, step=1, value=50)
|
||||
repetition_penalty = gr.Slider(label="Repetition penalty", minimum=1.0, maximum=2.0, step=0.05, value=1.2)
|
||||
|
||||
with gr.Column(scale=3):
|
||||
gr.Markdown("## Output", elem_id="output-title")
|
||||
raw_output = gr.Textbox(label="Raw Output Stream", interactive=False, lines=11, show_copy_button=True)
|
||||
with gr.Accordion("[Result.md]", open=False):
|
||||
formatted_output = gr.Markdown(label="Formatted Result")
|
||||
gr.Markdown("[Report any Bug/Issue here](https://huggingface.co/spaces/prithivMLmods/Multimodal-OCR3/discussions/1)")
|
||||
|
||||
image_submit.click(
|
||||
fn=generate_image,
|
||||
inputs=[image_query, image_upload, max_new_tokens, temperature, top_p, top_k, repetition_penalty],
|
||||
outputs=[raw_output, formatted_output]
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
demo.queue(max_size=50).launch(mcp_server=True, ssr_mode=False, show_error=True)
|
||||
```
|
||||
|
||||
## Implementation Example
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## If you intend to run dots.ocr with the original model path, implement and fix the issue through code-side actions.
|
||||
|
||||
```py
|
||||
CACHE_PATH = "./model_cache"
|
||||
if not os.path.exists(CACHE_PATH):
|
||||
os.makedirs(CACHE_PATH)
|
||||
|
||||
|
||||
model_path_d_local = snapshot_download(
|
||||
repo_id='rednote-hilab/dots.ocr',
|
||||
local_dir=os.path.join(CACHE_PATH, 'dots.ocr'),
|
||||
max_workers=20,
|
||||
local_dir_use_symlinks=False
|
||||
)
|
||||
|
||||
config_file_path = os.path.join(model_path_d_local, "configuration_dots.py")
|
||||
|
||||
if os.path.exists(config_file_path):
|
||||
with open(config_file_path, 'r') as f:
|
||||
input_code = f.read()
|
||||
|
||||
lines = input_code.splitlines()
|
||||
if "class DotsVLProcessor" in input_code and not any("attributes = " in line for line in lines):
|
||||
output_lines = []
|
||||
for line in lines:
|
||||
output_lines.append(line)
|
||||
if line.strip().startswith("class DotsVLProcessor"):
|
||||
output_lines.append(" attributes = [\"image_processor\", \"tokenizer\"]")
|
||||
|
||||
with open(config_file_path, 'w') as f:
|
||||
f.write('\n'.join(output_lines))
|
||||
print("Patched configuration_dots.py successfully.")
|
||||
|
||||
sys.path.append(model_path_d_local)
|
||||
|
||||
|
||||
# Load Dots.OCR from the local, patched directory
|
||||
MODEL_PATH_D = model_path_d_local
|
||||
processor_d = AutoProcessor.from_pretrained(MODEL_PATH_D, trust_remote_code=True)
|
||||
model_d = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_PATH_D,
|
||||
attn_implementation="flash_attention_2",
|
||||
torch_dtype=torch.bfloat16,
|
||||
device_map="auto",
|
||||
trust_remote_code=True
|
||||
).eval()
|
||||
```
|
||||
Reference in New Issue
Block a user