1038 lines
39 KiB
Python
1038 lines
39 KiB
Python
from .modeling_deepseekv2 import DeepseekV2Model, DeepseekV2ForCausalLM
|
||
from .configuration_deepseek_v2 import DeepseekV2Config
|
||
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
|
||
from typing import List, Optional, Tuple, Union
|
||
from transformers.cache_utils import Cache
|
||
import requests
|
||
from PIL import Image, ImageOps, ImageDraw, ImageFont
|
||
from io import BytesIO
|
||
import torch
|
||
import torch.nn as nn
|
||
from torch.nn import CrossEntropyLoss
|
||
from torchvision import transforms
|
||
from torchvision.transforms.functional import InterpolationMode
|
||
import os
|
||
from .deepencoder import build_sam_vit_b, build_clip_l, MlpProjector
|
||
from addict import Dict
|
||
from transformers import TextStreamer
|
||
from .conversation import get_conv_template
|
||
from abc import ABC
|
||
import math
|
||
import re
|
||
from tqdm import tqdm
|
||
import numpy as np
|
||
import time
|
||
|
||
|
||
def load_image(image_path):
|
||
|
||
try:
|
||
image = Image.open(image_path)
|
||
|
||
corrected_image = ImageOps.exif_transpose(image)
|
||
|
||
return corrected_image
|
||
|
||
except Exception as e:
|
||
print(f"error: {e}")
|
||
try:
|
||
return Image.open(image_path)
|
||
except:
|
||
return None
|
||
|
||
|
||
def re_match(text):
|
||
pattern = r'(<\|ref\|>(.*?)<\|/ref\|><\|det\|>(.*?)<\|/det\|>)'
|
||
matches = re.findall(pattern, text, re.DOTALL)
|
||
|
||
# pattern1 = r'<\|ref\|>.*?<\|/ref\|>\n'
|
||
# new_text1 = re.sub(pattern1, '', text, flags=re.DOTALL)
|
||
|
||
mathes_image = []
|
||
mathes_other = []
|
||
for a_match in matches:
|
||
if '<|ref|>image<|/ref|>' in a_match[0]:
|
||
mathes_image.append(a_match[0])
|
||
else:
|
||
mathes_other.append(a_match[0])
|
||
return matches, mathes_image, mathes_other
|
||
|
||
|
||
def extract_coordinates_and_label(ref_text, image_width, image_height):
|
||
|
||
try:
|
||
label_type = ref_text[1]
|
||
cor_list = eval(ref_text[2])
|
||
except Exception as e:
|
||
print(e)
|
||
return None
|
||
|
||
return (label_type, cor_list)
|
||
|
||
|
||
def draw_bounding_boxes(image, refs, ouput_path):
|
||
|
||
image_width, image_height = image.size
|
||
|
||
img_draw = image.copy()
|
||
draw = ImageDraw.Draw(img_draw)
|
||
|
||
overlay = Image.new('RGBA', img_draw.size, (0, 0, 0, 0))
|
||
draw2 = ImageDraw.Draw(overlay)
|
||
|
||
# try:
|
||
# except IOError:
|
||
# try:
|
||
# font = ImageFont.truetype("DejaVuSans.ttf", 20)
|
||
# except IOError:
|
||
font = ImageFont.load_default()
|
||
|
||
img_idx = 0
|
||
|
||
for i, ref in enumerate(refs):
|
||
try:
|
||
result = extract_coordinates_and_label(ref, image_width, image_height)
|
||
if result:
|
||
label_type, points_list = result
|
||
|
||
color = (np.random.randint(0, 200), np.random.randint(0, 200), np.random.randint(0, 255))
|
||
|
||
color_a = color + (20, )
|
||
for points in points_list:
|
||
x1, y1, x2, y2 = points
|
||
|
||
x1 = int(x1 / 999 * image_width)
|
||
y1 = int(y1 / 999 * image_height)
|
||
|
||
x2 = int(x2 / 999 * image_width)
|
||
y2 = int(y2 / 999 * image_height)
|
||
|
||
if label_type == 'image':
|
||
try:
|
||
cropped = image.crop((x1, y1, x2, y2))
|
||
cropped.save(f"{ouput_path}/images/{img_idx}.jpg")
|
||
except Exception as e:
|
||
print(e)
|
||
pass
|
||
img_idx += 1
|
||
|
||
try:
|
||
if label_type == 'title':
|
||
draw.rectangle([x1, y1, x2, y2], outline=color, width=4)
|
||
draw2.rectangle([x1, y1, x2, y2], fill=color_a, outline=(0, 0, 0, 0), width=1)
|
||
else:
|
||
draw.rectangle([x1, y1, x2, y2], outline=color, width=2)
|
||
draw2.rectangle([x1, y1, x2, y2], fill=color_a, outline=(0, 0, 0, 0), width=1)
|
||
text_x = x1
|
||
text_y = max(0, y1 - 15)
|
||
|
||
|
||
text_bbox = draw.textbbox((0, 0), label_type, font=font)
|
||
text_width = text_bbox[2] - text_bbox[0]
|
||
text_height = text_bbox[3] - text_bbox[1]
|
||
draw.rectangle([text_x, text_y, text_x + text_width, text_y + text_height],
|
||
fill=(255, 255, 255, 30))
|
||
|
||
draw.text((text_x, text_y), label_type, font=font, fill=color)
|
||
except:
|
||
pass
|
||
except:
|
||
continue
|
||
img_draw.paste(overlay, (0, 0), overlay)
|
||
return img_draw
|
||
|
||
|
||
def process_image_with_refs(image, ref_texts, output_path):
|
||
|
||
result_image = draw_bounding_boxes(image, ref_texts, output_path)
|
||
|
||
return result_image
|
||
|
||
|
||
|
||
|
||
|
||
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
|
||
best_ratio_diff = float('inf')
|
||
best_ratio = (1, 1)
|
||
area = width * height
|
||
for ratio in target_ratios:
|
||
target_aspect_ratio = ratio[0] / ratio[1]
|
||
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
|
||
if ratio_diff < best_ratio_diff:
|
||
best_ratio_diff = ratio_diff
|
||
best_ratio = ratio
|
||
elif ratio_diff == best_ratio_diff:
|
||
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
|
||
best_ratio = ratio
|
||
# print(f'width: {width}, height: {height}, best_ratio: {best_ratio}')
|
||
return best_ratio
|
||
|
||
|
||
def dynamic_preprocess(image, min_num=2, max_num=9, image_size=640, use_thumbnail=False):
|
||
orig_width, orig_height = image.size
|
||
aspect_ratio = orig_width / orig_height
|
||
|
||
# calculate the existing image aspect ratio
|
||
target_ratios = set(
|
||
(i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
|
||
i * j <= max_num and i * j >= min_num)
|
||
# print(target_ratios)
|
||
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
||
|
||
# find the closest aspect ratio to the target
|
||
target_aspect_ratio = find_closest_aspect_ratio(
|
||
aspect_ratio, target_ratios, orig_width, orig_height, image_size)
|
||
|
||
# print(target_aspect_ratio)
|
||
# calculate the target width and height
|
||
target_width = image_size * target_aspect_ratio[0]
|
||
target_height = image_size * target_aspect_ratio[1]
|
||
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
||
|
||
# resize the image
|
||
resized_img = image.resize((target_width, target_height))
|
||
processed_images = []
|
||
for i in range(blocks):
|
||
box = (
|
||
(i % (target_width // image_size)) * image_size,
|
||
(i // (target_width // image_size)) * image_size,
|
||
((i % (target_width // image_size)) + 1) * image_size,
|
||
((i // (target_width // image_size)) + 1) * image_size
|
||
)
|
||
# split the image
|
||
split_img = resized_img.crop(box)
|
||
processed_images.append(split_img)
|
||
assert len(processed_images) == blocks
|
||
if use_thumbnail and len(processed_images) != 1:
|
||
thumbnail_img = image.resize((image_size, image_size))
|
||
processed_images.append(thumbnail_img)
|
||
return processed_images, target_aspect_ratio
|
||
|
||
|
||
|
||
def normalize_transform(mean, std):
|
||
if mean is None and std is None:
|
||
transform = None
|
||
elif mean is None and std is not None:
|
||
mean = [0.] * len(std)
|
||
transform = transforms.Normalize(mean=mean, std=std)
|
||
elif mean is not None and std is None:
|
||
std = [1.] * len(mean)
|
||
transform = transforms.Normalize(mean=mean, std=std)
|
||
else:
|
||
transform = transforms.Normalize(mean=mean, std=std)
|
||
|
||
return transform
|
||
|
||
|
||
|
||
def format_messages(
|
||
conversations: List[Dict[str, str]],
|
||
sft_format: str = "deepseek",
|
||
system_prompt: str = "",
|
||
):
|
||
"""
|
||
Applies the SFT template to conversation.
|
||
|
||
Args:
|
||
conversations (List[Dict]): A List of messages.
|
||
sft_format (str, optional): The format of the SFT template to use. Defaults to "deepseek".
|
||
system_prompt (str, optional): The system prompt to use in the SFT template. Defaults to "".
|
||
|
||
Returns:
|
||
sft_prompt (str): The formatted text.
|
||
"""
|
||
|
||
conv = get_conv_template(sft_format)
|
||
conv.set_system_message(system_prompt)
|
||
for message in conversations:
|
||
conv.append_message(message["role"], message["content"].strip())
|
||
sft_prompt = conv.get_prompt().strip()
|
||
|
||
return sft_prompt
|
||
|
||
|
||
def text_encode(tokenizer, text: str, bos: bool = True, eos: bool = False):
|
||
t = tokenizer.encode(text, add_special_tokens=False)
|
||
bos_id = 0
|
||
eos_id = 1
|
||
if bos:
|
||
t = [bos_id] + t
|
||
if eos:
|
||
t = t + [eos_id]
|
||
|
||
return t
|
||
|
||
def load_pil_images(conversations: List[Dict[str, str]]) -> List[Image.Image]:
|
||
"""
|
||
|
||
Args:
|
||
conversations (List[Dict[str, str]]): the conversations with a list of messages. An example is :
|
||
[
|
||
{
|
||
"role": "User",
|
||
"content": "<image_placeholder>\nExtract all information from this image and convert them into markdown format.",
|
||
"images": ["./examples/table_datasets.png"]
|
||
},
|
||
{"role": "Assistant", "content": ""},
|
||
]
|
||
|
||
Returns:
|
||
pil_images (List[PIL.Image.Image]): the list of PIL images.
|
||
|
||
"""
|
||
|
||
pil_images = []
|
||
|
||
for message in conversations:
|
||
if "images" not in message:
|
||
continue
|
||
|
||
for image_path in message["images"]:
|
||
# print('----------------')
|
||
# print(image_path)
|
||
# print('----------------')
|
||
# exit()
|
||
|
||
# pil_img = Image.open(image_path)
|
||
pil_img = load_image(image_path)
|
||
pil_img = pil_img.convert("RGB")
|
||
pil_images.append(pil_img)
|
||
|
||
return pil_images
|
||
|
||
|
||
class BaseTransform(ABC):
|
||
|
||
def set_rng(self, *args, **kwargs):
|
||
pass
|
||
|
||
def __call__(self, *args, **kwargs) -> torch.Tensor:
|
||
pass
|
||
|
||
@property
|
||
def default_shape(self):
|
||
raise NotImplementedError
|
||
|
||
|
||
class BasicImageTransform(BaseTransform):
|
||
def __init__(
|
||
self,
|
||
mean: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5),
|
||
std: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5),
|
||
normalize: bool = True
|
||
):
|
||
self.mean = mean
|
||
self.std = std
|
||
|
||
transform_pipelines = [
|
||
transforms.ToTensor()
|
||
]
|
||
|
||
normalize = normalize_transform(mean, std) if normalize else nn.Identity()
|
||
if normalize is not None:
|
||
transform_pipelines.append(normalize)
|
||
|
||
self.transform = transforms.Compose(transform_pipelines)
|
||
|
||
def __call__(self, x):
|
||
x = self.transform(x)
|
||
return x
|
||
|
||
class NoEOSTextStreamer(TextStreamer):
|
||
def on_finalized_text(self, text: str, stream_end: bool = False):
|
||
|
||
eos_text = self.tokenizer.decode([self.tokenizer.eos_token_id], skip_special_tokens=False)
|
||
text = text.replace(eos_text, "\n")
|
||
print(text, flush=True, end="")
|
||
|
||
|
||
class DeepseekOCRConfig(DeepseekV2Config):
|
||
model_type = "DeepseekOCR"
|
||
|
||
class DeepseekOCRModel(DeepseekV2Model):
|
||
config_class = DeepseekOCRConfig
|
||
|
||
def __init__(self, config: DeepseekV2Config):
|
||
super(DeepseekOCRModel, self).__init__(config)
|
||
|
||
self.sam_model = build_sam_vit_b()
|
||
self.vision_model = build_clip_l()
|
||
# self.conv_2 = nn.Conv2d(in_channels=1024, out_channels=2048, kernel_size=2, stride=2)
|
||
n_embed = 1280
|
||
self.projector = MlpProjector(Dict(projector_type="linear", input_dim=2048, n_embed=n_embed))
|
||
embed_std = 1 / torch.sqrt(torch.tensor(n_embed, dtype=torch.float32))
|
||
self.image_newline = nn.Parameter(torch.randn(n_embed) * embed_std)
|
||
self.view_seperator = nn.Parameter(torch.randn(n_embed) * embed_std)
|
||
|
||
|
||
|
||
|
||
def forward(
|
||
self,
|
||
input_ids: torch.LongTensor = None,
|
||
attention_mask: Optional[torch.Tensor] = None,
|
||
position_ids: Optional[torch.LongTensor] = None,
|
||
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
||
inputs_embeds: Optional[torch.FloatTensor] = None,
|
||
use_cache: Optional[bool] = None,
|
||
output_attentions: Optional[bool] = None,
|
||
output_hidden_states: Optional[bool] = None,
|
||
images: Optional[torch.FloatTensor] = None,
|
||
images_seq_mask: Optional[torch.FloatTensor] = None,
|
||
images_spatial_crop: Optional[torch.FloatTensor] = None,
|
||
return_dict: Optional[bool] = None,
|
||
) -> Union[Tuple, BaseModelOutputWithPast]:
|
||
|
||
|
||
|
||
|
||
if inputs_embeds is None:
|
||
# inputs_embeds = self.embed_tokens(input_ids)
|
||
inputs_embeds = self.get_input_embeddings()(input_ids)
|
||
|
||
|
||
|
||
sam_model = getattr(self, 'sam_model', None)
|
||
# sam_model = self.sam_model
|
||
vision_model = getattr(self, 'vision_model', None)
|
||
|
||
|
||
|
||
if sam_model is not None and (input_ids.shape[1] != 1 or self.training) and torch.sum(images[0][1]).item() != 0:
|
||
|
||
idx = 0
|
||
|
||
# sam_model = torch.jit.script(sam_model)
|
||
|
||
# start_time = time.time()
|
||
for image, crop_shape in zip(images, images_spatial_crop):
|
||
images_in_this_batch = []
|
||
|
||
patches = image[0]
|
||
image_ori = image[1]
|
||
|
||
with torch.no_grad():
|
||
# with torch.inference_mode():
|
||
|
||
if torch.sum(patches).item() != 0:
|
||
# P, C, H, W = patches.shape
|
||
crop_flag = 1
|
||
local_features_1 = sam_model(patches)
|
||
|
||
local_features_2 = vision_model(patches, local_features_1)
|
||
# vit_time = time.time()
|
||
local_features = torch.cat((local_features_2[:, 1:], local_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
|
||
local_features = self.projector(local_features)
|
||
|
||
|
||
global_features_1 = sam_model(image_ori)
|
||
global_features_2 = vision_model(image_ori, global_features_1)
|
||
global_features = torch.cat((global_features_2[:, 1:], global_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
|
||
global_features = self.projector(global_features)
|
||
|
||
print('=====================')
|
||
print('BASE: ', global_features.shape)
|
||
print('PATCHES: ', local_features.shape)
|
||
print('=====================')
|
||
|
||
_, hw, n_dim = global_features.shape
|
||
h = w = int(hw ** 0.5)
|
||
|
||
_2, hw2, n_dim2 = local_features.shape
|
||
h2 = w2 = int(hw2 ** 0.5)
|
||
|
||
width_crop_num, height_crop_num = crop_shape[0], crop_shape[1]
|
||
|
||
global_features = global_features.view(h, w, n_dim)
|
||
|
||
global_features = torch.cat(
|
||
[global_features, self.image_newline[None, None, :].expand(h, 1, n_dim)], dim=1
|
||
)
|
||
|
||
global_features = global_features.view(-1, n_dim)
|
||
|
||
|
||
local_features = local_features.view(height_crop_num, width_crop_num, h2, w2, n_dim2).permute(0, 2, 1, 3, 4).reshape(height_crop_num*h2, width_crop_num*w2, n_dim2)
|
||
local_features = torch.cat(
|
||
[local_features, self.image_newline[None, None, :].expand(height_crop_num * h2, 1, n_dim2)], dim=1
|
||
)
|
||
local_features = local_features.view(-1, n_dim2)
|
||
|
||
global_local_features = torch.cat([local_features, global_features, self.view_seperator[None, :]], dim=0)
|
||
|
||
# end_time = time.time()
|
||
|
||
# print('sam: ', sam_time - start_time)
|
||
# print('vit: ', vit_time - sam_time)
|
||
# print('all: ', end_time - start_time)
|
||
|
||
# exit()
|
||
|
||
else:
|
||
global_features_1 = sam_model(image_ori)
|
||
global_features_2 = vision_model(image_ori, global_features_1)
|
||
global_features = torch.cat((global_features_2[:, 1:], global_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
|
||
global_features = self.projector(global_features)
|
||
print('=====================')
|
||
print('BASE: ', global_features.shape)
|
||
print('NO PATCHES')
|
||
print('=====================')
|
||
_, hw, n_dim = global_features.shape
|
||
h = w = int(hw ** 0.5)
|
||
|
||
|
||
global_features = global_features.view(h, w, n_dim)
|
||
|
||
global_features = torch.cat(
|
||
[global_features, self.image_newline[None, None, :].expand(h, 1, n_dim)], dim=1
|
||
)
|
||
|
||
global_features = global_features.view(-1, n_dim)
|
||
|
||
global_local_features = torch.cat([global_features, self.view_seperator[None, :]], dim=0)
|
||
|
||
images_in_this_batch.append(global_local_features)
|
||
|
||
|
||
# print(inputs_embeds.shape)
|
||
|
||
if images_in_this_batch:
|
||
images_in_this_batch = torch.cat(images_in_this_batch, dim=0)
|
||
# exit()
|
||
|
||
inputs_embeds[idx].masked_scatter_(images_seq_mask[idx].unsqueeze(-1).npu(), images_in_this_batch)
|
||
|
||
idx += 1
|
||
|
||
|
||
return super(DeepseekOCRModel, self).forward(
|
||
input_ids=None, attention_mask=attention_mask, past_key_values=past_key_values,
|
||
inputs_embeds=inputs_embeds, use_cache=use_cache, position_ids = position_ids,
|
||
output_attentions=output_attentions, output_hidden_states=output_hidden_states,
|
||
return_dict=return_dict
|
||
)
|
||
|
||
|
||
class DeepseekOCRForCausalLM(DeepseekV2ForCausalLM):
|
||
|
||
config_class = DeepseekOCRConfig
|
||
# supports_gradient_checkpointing = True
|
||
|
||
def __init__(self, config):
|
||
super(DeepseekV2ForCausalLM, self).__init__(config)
|
||
self.model = DeepseekOCRModel(config)
|
||
|
||
self.vocab_size = config.vocab_size
|
||
|
||
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
||
|
||
# self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
||
|
||
# Initialize weights and apply final processing
|
||
self.post_init()
|
||
|
||
def get_model(self):
|
||
return self.model
|
||
|
||
|
||
def forward(
|
||
self,
|
||
input_ids: torch.LongTensor = None,
|
||
attention_mask: Optional[torch.Tensor] = None,
|
||
position_ids: Optional[torch.LongTensor] = None,
|
||
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
||
inputs_embeds: Optional[torch.FloatTensor] = None,
|
||
labels: Optional[torch.LongTensor] = None,
|
||
use_cache: Optional[bool] = None,
|
||
output_attentions: Optional[bool] = None,
|
||
output_hidden_states: Optional[bool] = None,
|
||
images: Optional[torch.FloatTensor] = None,
|
||
images_seq_mask: Optional[torch.FloatTensor] = None,
|
||
images_spatial_crop: Optional[torch.FloatTensor] = None,
|
||
return_dict: Optional[bool] = None,
|
||
|
||
) -> Union[Tuple, CausalLMOutputWithPast]:
|
||
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
||
output_hidden_states = (
|
||
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
||
)
|
||
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
||
|
||
|
||
|
||
outputs = self.model(
|
||
input_ids=input_ids,
|
||
past_key_values=past_key_values,
|
||
attention_mask=attention_mask,
|
||
position_ids=position_ids,
|
||
inputs_embeds=inputs_embeds,
|
||
use_cache=use_cache,
|
||
output_attentions=output_attentions,
|
||
output_hidden_states=output_hidden_states,
|
||
images=images,
|
||
images_seq_mask = images_seq_mask,
|
||
images_spatial_crop = images_spatial_crop,
|
||
return_dict=return_dict
|
||
|
||
)
|
||
|
||
|
||
|
||
# print(transformer_outputs)
|
||
|
||
hidden_states = outputs[0]
|
||
logits = self.lm_head(hidden_states)
|
||
logits = logits.float()
|
||
|
||
# logits
|
||
|
||
loss = None
|
||
if labels is not None:
|
||
# Shift so that tokens < n predict n
|
||
shift_logits = logits[..., :-1, :].contiguous()
|
||
shift_labels = labels[..., 1:].contiguous()
|
||
# Flatten the tokens
|
||
loss_fct = CrossEntropyLoss()
|
||
shift_logits = shift_logits.view(-1, self.config.vocab_size)
|
||
shift_labels = shift_labels.view(-1)
|
||
# Enable model parallelism
|
||
shift_labels = shift_labels.to(shift_logits.device)
|
||
loss = loss_fct(shift_logits, shift_labels)
|
||
|
||
if not return_dict:
|
||
output = (logits,) + outputs[1:]
|
||
return (loss,) + output if loss is not None else output
|
||
|
||
return CausalLMOutputWithPast(
|
||
loss=loss,
|
||
logits=logits,
|
||
past_key_values=outputs.past_key_values,
|
||
hidden_states=outputs.hidden_states,
|
||
attentions=outputs.attentions,
|
||
)
|
||
|
||
|
||
def prepare_inputs_for_generation(
|
||
self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
|
||
):
|
||
# Omit tokens covered by past_key_values
|
||
past_length = 0
|
||
if past_key_values is not None:
|
||
if isinstance(past_key_values, Cache):
|
||
cache_length = past_key_values.get_seq_length()
|
||
past_length = past_key_values.seen_tokens
|
||
max_cache_length = past_key_values.get_max_length()
|
||
else:
|
||
cache_length = past_length = past_key_values[0][0].shape[2]
|
||
max_cache_length = None
|
||
|
||
# Keep only the unprocessed tokens:
|
||
# 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
|
||
# some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
|
||
# input)
|
||
if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
|
||
input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
|
||
# 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
|
||
# input_ids based on the past_length.
|
||
elif past_length < input_ids.shape[1]:
|
||
input_ids = input_ids[:, past_length:]
|
||
# 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
|
||
|
||
# If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
|
||
if (
|
||
max_cache_length is not None
|
||
and attention_mask is not None
|
||
and cache_length + input_ids.shape[1] > max_cache_length
|
||
):
|
||
attention_mask = attention_mask[:, -max_cache_length:]
|
||
|
||
position_ids = kwargs.get("position_ids", None)
|
||
if attention_mask is not None and position_ids is None:
|
||
# create position_ids on the fly for batch generation
|
||
position_ids = attention_mask.long().cumsum(-1) - 1
|
||
position_ids.masked_fill_(attention_mask == 0, 1)
|
||
if past_key_values:
|
||
position_ids = position_ids[:, -input_ids.shape[1] :]
|
||
|
||
# if self.generation_config.cache_implementation == "static":
|
||
# # generation with static cache
|
||
# cache_position = kwargs.get("cache_position", None)
|
||
# if cache_position is None:
|
||
# past_length = 0
|
||
# else:
|
||
# past_length = cache_position[-1] + 1
|
||
# input_ids = input_ids[:, past_length:]
|
||
# position_ids = position_ids[:, past_length:]
|
||
|
||
# TODO @gante we should only keep a `cache_position` in generate, and do +=1.
|
||
# same goes for position ids. Could also help with continued generation.
|
||
cache_position = torch.arange(past_length, past_length + position_ids.shape[-1], device=position_ids.device)
|
||
|
||
# if `inputs_embeds` are passed, we only want to use them in the 1st generation step
|
||
if inputs_embeds is not None and past_key_values is None:
|
||
model_inputs = {"inputs_embeds": inputs_embeds}
|
||
else:
|
||
model_inputs = {"input_ids": input_ids}
|
||
|
||
model_inputs.update(
|
||
{
|
||
"position_ids": position_ids,
|
||
"past_key_values": past_key_values,
|
||
"use_cache": kwargs.get("use_cache"),
|
||
"attention_mask": attention_mask,
|
||
"images": kwargs.get("images", None),
|
||
"images_seq_mask": kwargs.get("images_seq_mask", None),
|
||
"images_spatial_crop": kwargs.get("images_spatial_crop", None),
|
||
}
|
||
)
|
||
return model_inputs
|
||
|
||
|
||
def disable_torch_init(self):
|
||
"""
|
||
Disable the redundant torch default initialization to accelerate model creation.
|
||
"""
|
||
import torch
|
||
setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
|
||
setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
|
||
|
||
|
||
|
||
def infer(self, tokenizer, prompt='', image_file='', output_path = '', base_size=1024, image_size=640, crop_mode=True, test_compress=False, save_results=False, eval_mode=False):
|
||
self.disable_torch_init()
|
||
|
||
os.makedirs(output_path, exist_ok=True)
|
||
os.makedirs(f'{output_path}/images', exist_ok=True)
|
||
|
||
if prompt and image_file:
|
||
conversation = [
|
||
{
|
||
"role": "<|User|>",
|
||
# "content": "<image>\n<|grounding|>Given the layout of the image. ",
|
||
"content": f'{prompt}',
|
||
# "content": "君不见黄河之水天上来的下一句是什么？",
|
||
# "content": "<image>\nFree OCR. ",
|
||
# "content": "<image>\nParse the figure. ",
|
||
# "content": "<image>\nExtract the text in the image. ",
|
||
"images": [f'{image_file}'],
|
||
},
|
||
{"role": "<|Assistant|>", "content": ""},
|
||
]
|
||
|
||
elif prompt:
|
||
conversation = [
|
||
{
|
||
"role": "<|User|>",
|
||
# "content": "<image>\n<|grounding|>Given the layout of the image. ",
|
||
"content": f'{prompt}',
|
||
# "content": "君不见黄河之水天上来的下一句是什么？",
|
||
# "content": "<image>\nFree OCR. ",
|
||
# "content": "<image>\nParse the figure. ",
|
||
# "content": "<image>\nExtract the text in the image. ",
|
||
# "images": [f'{image_file}'],
|
||
},
|
||
{"role": "<|Assistant|>", "content": ""},
|
||
]
|
||
else:
|
||
assert False, f'prompt is none!'
|
||
|
||
prompt = format_messages(conversations=conversation, sft_format='plain', system_prompt='')
|
||
|
||
patch_size = 16
|
||
downsample_ratio = 4
|
||
images = load_pil_images(conversation)
|
||
|
||
valid_img_tokens = 0
|
||
ratio = 1
|
||
|
||
image_draw = images[0].copy()
|
||
|
||
w,h = image_draw.size
|
||
# print(w, h)
|
||
ratio = 1 - ((max(w, h) - min(w, h)) / (max(w, h)))
|
||
|
||
|
||
image_transform=BasicImageTransform(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5), normalize=True)
|
||
images_seq_mask = []
|
||
|
||
image_token = '<image>'
|
||
image_token_id = 128815
|
||
text_splits = prompt.split(image_token)
|
||
|
||
images_list, images_crop_list, images_seq_mask = [], [], []
|
||
tokenized_str = []
|
||
images_spatial_crop = []
|
||
for text_sep, image in zip(text_splits, images):
|
||
|
||
tokenized_sep = text_encode(tokenizer, text_sep, bos=False, eos=False)
|
||
tokenized_str += tokenized_sep
|
||
images_seq_mask += [False] * len(tokenized_sep)
|
||
|
||
if crop_mode:
|
||
|
||
if image.size[0] <= 640 and image.size[1] <= 640:
|
||
crop_ratio = [1, 1]
|
||
|
||
else:
|
||
if crop_mode:
|
||
# best_width, best_height = select_best_resolution(image.size, self.candidate_resolutions)
|
||
images_crop_raw, crop_ratio = dynamic_preprocess(image)
|
||
else:
|
||
# best_width, best_height = self.image_size, self.image_size
|
||
crop_ratio = [1, 1]
|
||
|
||
"""process the global view"""
|
||
# image = image.resize((base_size, base_size))
|
||
global_view = ImageOps.pad(image, (base_size, base_size),
|
||
color=tuple(int(x * 255) for x in image_transform.mean))
|
||
|
||
if base_size == 1024:
|
||
valid_img_tokens += int(256 * ratio)
|
||
elif base_size == 1280:
|
||
valid_img_tokens += int(400 * ratio)
|
||
# elif base_size == 640:
|
||
# valid_img_tokens += int(100 * ratio)
|
||
|
||
|
||
|
||
|
||
|
||
images_list.append(image_transform(global_view).to(torch.bfloat16))
|
||
|
||
# global_view_tensor = image_transform(global_view).to(torch.bfloat16)
|
||
|
||
width_crop_num, height_crop_num = crop_ratio
|
||
|
||
images_spatial_crop.append([width_crop_num, height_crop_num])
|
||
|
||
|
||
if width_crop_num > 1 or height_crop_num > 1:
|
||
"""process the local views"""
|
||
|
||
for i in range(len(images_crop_raw)):
|
||
images_crop_list.append(image_transform(images_crop_raw[i]).to(torch.bfloat16))
|
||
|
||
if image_size == 640:
|
||
valid_img_tokens += len(images_crop_list) * 100
|
||
|
||
num_queries = math.ceil((image_size // patch_size) / downsample_ratio)
|
||
num_queries_base = math.ceil((base_size // patch_size) / downsample_ratio)
|
||
|
||
|
||
|
||
"""add image tokens"""
|
||
|
||
|
||
|
||
tokenized_image = ([image_token_id] * num_queries_base + [image_token_id]) * num_queries_base
|
||
tokenized_image += [image_token_id]
|
||
if width_crop_num > 1 or height_crop_num > 1:
|
||
tokenized_image += ([image_token_id] * (num_queries * width_crop_num) + [image_token_id]) * (
|
||
num_queries * height_crop_num)
|
||
tokenized_str += tokenized_image
|
||
images_seq_mask += [True] * len(tokenized_image)
|
||
# num_image_tokens.append(len(tokenized_image))
|
||
|
||
else:
|
||
# best_width, best_height = self.image_size, self.image_size
|
||
# print(image.size, (best_width, best_height)) # check the select_best_resolutions func
|
||
|
||
"""process the global view"""
|
||
if image_size <= 640:
|
||
print('directly resize')
|
||
image = image.resize((image_size, image_size))
|
||
# else:
|
||
global_view = ImageOps.pad(image, (image_size, image_size),
|
||
color=tuple(int(x * 255) for x in image_transform.mean))
|
||
images_list.append(image_transform(global_view).to(torch.bfloat16))
|
||
|
||
if base_size == 1024:
|
||
valid_img_tokens += int(256 * ratio)
|
||
elif base_size == 1280:
|
||
valid_img_tokens += int(400 * ratio)
|
||
elif base_size == 640:
|
||
valid_img_tokens += int(100 * 1)
|
||
elif base_size == 512:
|
||
valid_img_tokens += int(64 * 1)
|
||
|
||
width_crop_num, height_crop_num = 1, 1
|
||
|
||
images_spatial_crop.append([width_crop_num, height_crop_num])
|
||
|
||
|
||
"""add image tokens"""
|
||
num_queries = math.ceil((image_size // patch_size) / downsample_ratio)
|
||
|
||
tokenized_image = ([image_token_id] * num_queries + [image_token_id]) * num_queries
|
||
tokenized_image += [image_token_id]
|
||
# tokenized_image += ([self.image_token_id] * (num_queries * width_crop_num) + [self.image_token_id]) * (
|
||
# num_queries * height_crop_num)
|
||
tokenized_str += tokenized_image
|
||
images_seq_mask += [True] * len(tokenized_image)
|
||
# num_image_tokens.append(len(tokenized_image))
|
||
|
||
|
||
"""process the last text split"""
|
||
tokenized_sep = text_encode(tokenizer, text_splits[-1], bos=False, eos=False)
|
||
tokenized_str += tokenized_sep
|
||
images_seq_mask += [False] * len(tokenized_sep)
|
||
|
||
"""add the bos tokens"""
|
||
bos_id = 0
|
||
tokenized_str = [bos_id] + tokenized_str
|
||
images_seq_mask = [False] + images_seq_mask
|
||
|
||
|
||
|
||
input_ids = torch.LongTensor(tokenized_str)
|
||
|
||
|
||
|
||
|
||
images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)
|
||
|
||
|
||
if len(images_list) == 0:
|
||
images_ori = torch.zeros((1, 3, image_size, image_size))
|
||
images_spatial_crop = torch.zeros((1, 2), dtype=torch.long)
|
||
images_crop = torch.zeros((1, 3, base_size, base_size))
|
||
|
||
else:
|
||
images_ori = torch.stack(images_list, dim=0)
|
||
images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)
|
||
if images_crop_list:
|
||
images_crop = torch.stack(images_crop_list, dim=0)
|
||
else:
|
||
images_crop = torch.zeros((1, 3, base_size, base_size))
|
||
|
||
|
||
|
||
if not eval_mode:
|
||
streamer = NoEOSTextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=False)
|
||
with torch.autocast("cuda", dtype=torch.bfloat16):
|
||
with torch.no_grad():
|
||
output_ids = self.generate(
|
||
input_ids.unsqueeze(0).npu(),
|
||
images=[(images_crop.npu(), images_ori.npu())],
|
||
images_seq_mask = images_seq_mask.unsqueeze(0).npu(),
|
||
images_spatial_crop = images_spatial_crop,
|
||
# do_sample=False,
|
||
# num_beams = 1,
|
||
temperature=0.0,
|
||
eos_token_id=tokenizer.eos_token_id,
|
||
streamer=streamer,
|
||
max_new_tokens=8192,
|
||
no_repeat_ngram_size = 20,
|
||
use_cache = True
|
||
)
|
||
|
||
else:
|
||
with torch.autocast("cuda", dtype=torch.bfloat16):
|
||
with torch.no_grad():
|
||
output_ids = self.generate(
|
||
input_ids.unsqueeze(0).npu(),
|
||
images=[(images_crop.npu(), images_ori.npu())],
|
||
images_seq_mask = images_seq_mask.unsqueeze(0).npu(),
|
||
images_spatial_crop = images_spatial_crop,
|
||
# do_sample=False,
|
||
# num_beams = 1,
|
||
temperature=0.0,
|
||
eos_token_id=tokenizer.eos_token_id,
|
||
max_new_tokens=8192,
|
||
no_repeat_ngram_size = 35,
|
||
use_cache = True
|
||
)
|
||
|
||
|
||
if '<image>' in conversation[0]['content'] and eval_mode:
|
||
outputs = tokenizer.decode(output_ids[0, input_ids.unsqueeze(0).npu().shape[1]:])
|
||
stop_str = '<｜end▁of▁sentence｜>'
|
||
if outputs.endswith(stop_str):
|
||
outputs = outputs[:-len(stop_str)]
|
||
# re_match
|
||
outputs = outputs.strip()
|
||
|
||
return outputs
|
||
|
||
if '<image>' in conversation[0]['content'] and test_compress:
|
||
outputs = tokenizer.decode(output_ids[0, input_ids.unsqueeze(0).npu().shape[1]:])
|
||
pure_texts_outputs_token_length = len(text_encode(tokenizer, outputs, bos=False, eos=False))
|
||
print('='*50)
|
||
print('image size: ', (w, h))
|
||
print('valid image tokens: ', int(valid_img_tokens))
|
||
print('output texts tokens (valid): ', pure_texts_outputs_token_length)
|
||
print('compression ratio: ', round(pure_texts_outputs_token_length/valid_img_tokens, 2))
|
||
print('='*50)
|
||
|
||
|
||
if '<image>' in conversation[0]['content'] and save_results:
|
||
outputs = tokenizer.decode(output_ids[0, input_ids.unsqueeze(0).npu().shape[1]:])
|
||
stop_str = '<｜end▁of▁sentence｜>'
|
||
|
||
print('='*15 + 'save results:' + '='*15)
|
||
|
||
# # # # conv.messages[-1][-1] = outputs
|
||
if outputs.endswith(stop_str):
|
||
outputs = outputs[:-len(stop_str)]
|
||
outputs = outputs.strip()
|
||
|
||
matches_ref, matches_images, mathes_other = re_match(outputs)
|
||
# print(matches_ref)
|
||
result = process_image_with_refs(image_draw, matches_ref, output_path)
|
||
|
||
|
||
for idx, a_match_image in enumerate(tqdm(matches_images, desc="image")):
|
||
outputs = outputs.replace(a_match_image, ' + '.jpg)\n')
|
||
|
||
for idx, a_match_other in enumerate(tqdm(mathes_other, desc="other")):
|
||
outputs = outputs.replace(a_match_other, '').replace('\\coloneqq', ':=').replace('\\eqqcolon', '=:')
|
||
|
||
|
||
# if 'structural formula' in conversation[0]['content']:
|
||
# outputs = '<smiles>' + outputs + '</smiles>'
|
||
with open(f'{output_path}/result.mmd', 'w', encoding = 'utf-8') as afile:
|
||
afile.write(outputs)
|
||
|
||
if 'line_type' in outputs:
|
||
import matplotlib.pyplot as plt
|
||
lines = eval(outputs)['Line']['line']
|
||
|
||
line_type = eval(outputs)['Line']['line_type']
|
||
# print(lines)
|
||
|
||
endpoints = eval(outputs)['Line']['line_endpoint']
|
||
|
||
fig, ax = plt.subplots(figsize=(3,3), dpi=200)
|
||
ax.set_xlim(-15, 15)
|
||
ax.set_ylim(-15, 15)
|
||
|
||
for idx, line in enumerate(lines):
|
||
try:
|
||
p0 = eval(line.split(' -- ')[0])
|
||
p1 = eval(line.split(' -- ')[-1])
|
||
|
||
if line_type[idx] == '--':
|
||
ax.plot([p0[0], p1[0]], [p0[1], p1[1]], linewidth=0.8, color='k')
|
||
else:
|
||
ax.plot([p0[0], p1[0]], [p0[1], p1[1]], linewidth = 0.8, color = 'k')
|
||
|
||
ax.scatter(p0[0], p0[1], s=5, color = 'k')
|
||
ax.scatter(p1[0], p1[1], s=5, color = 'k')
|
||
except:
|
||
pass
|
||
|
||
for endpoint in endpoints:
|
||
|
||
label = endpoint.split(': ')[0]
|
||
(x, y) = eval(endpoint.split(': ')[1])
|
||
ax.annotate(label, (x, y), xytext=(1, 1), textcoords='offset points',
|
||
fontsize=5, fontweight='light')
|
||
|
||
|
||
plt.savefig(f'{output_path}/geo.jpg')
|
||
plt.close()
|
||
|
||
result.save(f"{output_path}/result_with_boxes.jpg")
|