From cf98de96b6bd8dbcc9545c77a6d72f451f232f82 Mon Sep 17 00:00:00 2001
From: lddfym <1799858408@qq.com>
Date: Thu, 14 Aug 2025 11:07:11 +0800
Subject: [PATCH 01/16] add qwen-2.5-vl processor
---
fastdeploy/input/ernie_vl_processor.py | 27 +-
fastdeploy/input/mm_processor/__init__.py | 3 +-
.../image_preprocessor_adaptive.py | 36 +-
fastdeploy/input/mm_processor/process.py | 313 +++------
.../input/mm_processor/process_video.py | 167 +----
fastdeploy/input2/__init__.py | 15 +
fastdeploy/input2/ernie_processor.py | 425 +++++++++++++
fastdeploy/input2/ernie_tokenizer.py | 394 ++++++++++++
fastdeploy/input2/ernie_vl_processor.py | 285 +++++++++
fastdeploy/input2/mm_processor/__init__.py | 23 +
.../image_preprocessor/__init__.py | 20 +
.../get_image_preprocessor.py | 34 +
.../image_preprocessor_adaptive.py | 587 +++++++++++++++++
fastdeploy/input2/mm_processor/process.py | 512 +++++++++++++++
.../input2/mm_processor/process_video.py | 205 ++++++
.../mm_processor/tokenizer/__init__.py | 0
.../mm_processor/tokenizer/tokenizer_vl.py | 0
.../mm_processor/utils/Roboto-Regular.ttf | Bin 0 -> 146004 bytes
.../input2/mm_processor/utils/__init__.py | 15 +
.../input2/mm_processor/utils/io_utils.py | 264 ++++++++
.../mm_processor/utils/render_timestamp.py | 103 +++
.../input2/mm_processor/utils/video_utils.py | 83 +++
fastdeploy/input2/preprocess.py | 101 +++
fastdeploy/input2/text_processor.py | 602 ++++++++++++++++++
24 files changed, 3805 insertions(+), 409 deletions(-)
create mode 100644 fastdeploy/input2/__init__.py
create mode 100644 fastdeploy/input2/ernie_processor.py
create mode 100644 fastdeploy/input2/ernie_tokenizer.py
create mode 100644 fastdeploy/input2/ernie_vl_processor.py
create mode 100644 fastdeploy/input2/mm_processor/__init__.py
create mode 100644 fastdeploy/input2/mm_processor/image_preprocessor/__init__.py
create mode 100644 fastdeploy/input2/mm_processor/image_preprocessor/get_image_preprocessor.py
create mode 100644 fastdeploy/input2/mm_processor/image_preprocessor/image_preprocessor_adaptive.py
create mode 100644 fastdeploy/input2/mm_processor/process.py
create mode 100644 fastdeploy/input2/mm_processor/process_video.py
rename fastdeploy/{input => input2}/mm_processor/tokenizer/__init__.py (100%)
rename fastdeploy/{input => input2}/mm_processor/tokenizer/tokenizer_vl.py (100%)
create mode 100644 fastdeploy/input2/mm_processor/utils/Roboto-Regular.ttf
create mode 100644 fastdeploy/input2/mm_processor/utils/__init__.py
create mode 100644 fastdeploy/input2/mm_processor/utils/io_utils.py
create mode 100644 fastdeploy/input2/mm_processor/utils/render_timestamp.py
create mode 100644 fastdeploy/input2/mm_processor/utils/video_utils.py
create mode 100644 fastdeploy/input2/preprocess.py
create mode 100644 fastdeploy/input2/text_processor.py
diff --git a/fastdeploy/input/ernie_vl_processor.py b/fastdeploy/input/ernie_vl_processor.py
index 63ae5bc310..d7ce66e932 100644
--- a/fastdeploy/input/ernie_vl_processor.py
+++ b/fastdeploy/input/ernie_vl_processor.py
@@ -43,8 +43,9 @@ def __init__(
image_preprocessor_name=preprocessor_path,
**processor_kwargs,
)
- self.ernie_processor.eval()
- self.image_patch_id = self.ernie_processor.image_patch_id
+
+ # self.ernie_processor.eval()
+ self.image_patch_id = self.ernie_processor.image_token_id
self.spatial_conv_size = self.ernie_processor.spatial_conv_size
self.decode_status = dict()
@@ -249,14 +250,20 @@ def append_generated_tokens(self, multimodal_inputs, generated_token_ids):
def pack_outputs(self, outs):
# Stack or nullify image-related fields
- if not outs["images"]:
- outs["images"] = None
- outs["grid_thw"] = None
- outs["image_type_ids"] = None
- else:
- outs["images"] = np.vstack(outs["images"])
- outs["grid_thw"] = np.vstack(outs["grid_thw"])
- outs["image_type_ids"] = np.array(outs["image_type_ids"])
+ # if not outs["images"]:
+ # outs["images"] = None
+ # outs["grid_thw"] = None
+ # outs["image_type_ids"] = None
+ # else:
+ # outs["images"] = np.vstack(outs["images"])
+ # outs["grid_thw"] = np.vstack(outs["grid_thw"])
+ # outs["image_type_ids"] = np.array(outs["image_type_ids"])
+
+ outs["images"] = np.array(outs["images"])
+ outs["grid_thw"] = np.array(outs["grid_thw"])
+ # outs["pixel_values_videos"] = np.array(outs["pixel_values_videos"])
+ # outs["video_grid_thw"] = np.array(outs["video_grid_thw"])
+ outs["image_type_ids"] = np.array(outs["image_type_ids"])
outs["image_patch_id"] = self.image_patch_id
# Convert lists to arrays
diff --git a/fastdeploy/input/mm_processor/__init__.py b/fastdeploy/input/mm_processor/__init__.py
index ba59bc1654..5a97e41863 100644
--- a/fastdeploy/input/mm_processor/__init__.py
+++ b/fastdeploy/input/mm_processor/__init__.py
@@ -14,10 +14,9 @@
# limitations under the License.
"""
-from .process import IDS_TYPE_FLAG, DataProcessor, fancy_print
+from .process import IDS_TYPE_FLAG, DataProcessor
__all__ = [
"DataProcessor",
- "fancy_print",
"IDS_TYPE_FLAG",
]
diff --git a/fastdeploy/input/mm_processor/image_preprocessor/image_preprocessor_adaptive.py b/fastdeploy/input/mm_processor/image_preprocessor/image_preprocessor_adaptive.py
index 15b15a4d22..c86d8046e3 100644
--- a/fastdeploy/input/mm_processor/image_preprocessor/image_preprocessor_adaptive.py
+++ b/fastdeploy/input/mm_processor/image_preprocessor/image_preprocessor_adaptive.py
@@ -153,7 +153,7 @@ class AdaptiveImageProcessor(BaseImageProcessor):
The max pixels of the image to resize the image.
patch_size (`int`, *optional*, defaults to 14):
The spacial patch size of the vision encoder.
- temporal_conv_size (`int`, *optional*, defaults to 2):
+ temporal_patch_size (`int`, *optional*, defaults to 2):
The temporal conv size in resampler.
merge_size (`int`, *optional*, defaults to 2):
The merge size of the vision encoder to llm encoder.
@@ -179,7 +179,7 @@ def __init__(
min_pixels: int = 56 * 56,
max_pixels: int = 28 * 28 * 1280,
patch_size: int = 14,
- temporal_conv_size: int = 2,
+ temporal_patch_size: int = 2,
merge_size: int = 2,
**kwargs,
) -> None:
@@ -195,7 +195,7 @@ def __init__(
self.min_pixels = min_pixels
self.max_pixels = max_pixels
self.patch_size = patch_size
- self.temporal_conv_size = temporal_conv_size
+ self.temporal_patch_size = temporal_patch_size
self.merge_size = merge_size
self.size = {"min_pixels": min_pixels, "max_pixels": max_pixels}
self.do_convert_rgb = do_convert_rgb
@@ -332,10 +332,18 @@ def _preprocess(
resample=resample,
data_format=input_data_format,
)
+
+ if do_rescale and do_normalize:
+ image_mean = np.array(image_mean, dtype=np.float32) * (1.0 / rescale_factor)
+ image_std = np.array(image_std, dtype=np.float32) * (1.0 / rescale_factor)
+ do_rescale = False
+
if do_rescale:
+ image = image.astype("float32")
image = rescale(image, scale=rescale_factor, data_format=input_data_format)
if do_normalize:
+ image = image.astype("float32")
image = normalize(
image=image,
mean=image_mean,
@@ -344,14 +352,21 @@ def _preprocess(
)
image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) # [C, H, W]
-
processed_images.append(image)
+
patches = np.array(processed_images)
+ if patches.shape[0] % self.temporal_patch_size != 0:
+ repeats = np.repeat(
+ patches[-1][np.newaxis], self.temporal_patch_size - (patches.shape[0] % self.temporal_patch_size), axis=0
+ )
+ patches = np.concatenate([patches, repeats], axis=0)
+
if data_format == ChannelDimension.LAST:
patches = patches.transpose([0, 3, 1, 2])
+
+ grid_t, channel = patches.shape[:2]
+ grid_t = grid_t // self.temporal_patch_size
- channel = patches.shape[1] # [time, C, H, W]
- grid_t = patches.shape[0]
grid_h, grid_w = (
resized_height // self.patch_size,
resized_width // self.patch_size,
@@ -359,6 +374,7 @@ def _preprocess(
patches = patches.reshape(
[
grid_t,
+ self.temporal_patch_size,
channel,
grid_h // self.merge_size,
self.merge_size,
@@ -368,15 +384,15 @@ def _preprocess(
self.patch_size,
]
)
- # [grid_t, grid_h/merge_size, grid_w/merge_size, merge_size, merge_size, C, psz, psz]
- patches = patches.transpose([0, 2, 5, 3, 6, 1, 4, 7])
+ # [grid_t, temporal_patch_size, grid_h/merge_size, grid_w/merge_size, merge_size, merge_size, C, psz, psz]
+ patches = patches.transpose([0, 3, 6, 4, 7, 2, 1, 5, 8])
flatten_patches = patches.reshape(
[
grid_t * grid_h * grid_w,
- channel * self.patch_size * self.patch_size,
+ channel * self.temporal_patch_size * self.patch_size * self.patch_size,
]
- ) # [grid_t * grid_h * grid_w, C * psz * psz]
+ )
return flatten_patches, (grid_t, grid_h, grid_w)
diff --git a/fastdeploy/input/mm_processor/process.py b/fastdeploy/input/mm_processor/process.py
index ea2559a0fe..bcdff21313 100644
--- a/fastdeploy/input/mm_processor/process.py
+++ b/fastdeploy/input/mm_processor/process.py
@@ -16,9 +16,6 @@
"""
""" process.py """
-import copy
-import os
-from collections import defaultdict
from typing import Any, Dict, List, Union
import numpy as np
@@ -26,76 +23,37 @@
from PIL import Image
from fastdeploy.entrypoints.chat_utils import parse_chat_messages
-from fastdeploy.input.ernie_tokenizer import ErnieBotTokenizer
from fastdeploy.utils import data_processor_logger
+from paddleformers.transformers import AutoTokenizer
+
from .image_preprocessor.image_preprocessor_adaptive import AdaptiveImageProcessor
-from .process_video import read_frames_decord, read_video_decord
-from .utils.render_timestamp import render_frame_timestamp
+from .process_video import read_video_decord
IDS_TYPE_FLAG = {"text": 0, "image": 1, "video": 2, "audio": 3}
-def fancy_print(input_ids, tokenizer, image_patch_id=None):
- """
- input_ids: input_ids
- tokenizer: the tokenizer of models
- """
- i = 0
- res = ""
- text_ids = []
- real_image_token_len = 0
- while i < len(input_ids):
- if input_ids[i] == image_patch_id:
- if len(text_ids) > 0:
- res += tokenizer.decode(text_ids)
- text_ids = []
-
- real_image_token_len += 1
- else:
- if real_image_token_len != 0:
- res += f"<|IMAGE@{real_image_token_len}|>"
- real_image_token_len = 0
-
- text_ids.append(input_ids[i])
-
- i += 1
- if len(text_ids) > 0:
-
- res += tokenizer.decode(text_ids)
- text_ids = []
- return res
-
-
class DataProcessor:
"""
Processes multimodal chat messages into model-ready inputs,
handling text, images, and videos with 3D positional embeddings.
"""
- CLS_TOKEN = "<|begin_of_sentence|>"
- SEP_TOKEN = "<|end_of_sentence|>"
- EOS_TOKEN = ""
- IMG_START = "<|IMAGE_START|>"
- IMG_END = "<|IMAGE_END|>"
- VID_START = "<|VIDEO_START|>"
- VID_END = "<|VIDEO_END|>"
-
def __init__(
self,
tokenizer_name: str,
image_preprocessor_name: str,
spatial_conv_size: int = 2,
temporal_conv_size: int = 2,
- image_min_pixels: int = 4 * 28 * 28,
- image_max_pixels: int = 6177 * 28 * 28,
- video_min_pixels: int = 299 * 28 * 28,
- video_max_pixels: int = 1196 * 28 * 28,
- video_target_frames: int = -1,
- video_frames_sample: str = "leading",
- video_max_frames: int = 180,
- video_min_frames: int = 16,
- video_fps: int = 2,
+ image_min_pixels: int = 3136,
+ image_max_pixels: int = 12845056,
+ video_min_pixels: int = 3136,
+ video_max_pixels: int = 12845056,
+ # video_target_frames: int = -1,
+ # video_frames_sample: str = "leading",
+ # video_max_frames: int = 180,
+ # video_min_frames: int = 16,
+ # video_fps: int = 2,
**kwargs,
) -> None:
# Tokenizer and image preprocessor
@@ -115,28 +73,26 @@ def __init__(
self.video_max_pixels = video_max_pixels
# Video sampling parameters
- self.target_frames = video_target_frames
- self.frames_sample = video_frames_sample
- self.max_frames = video_max_frames
- self.min_frames = video_min_frames
- self.fps = video_fps
+ # self.target_frames = video_target_frames
+ # self.frames_sample = video_frames_sample
+ # self.max_frames = video_max_frames
+ # self.min_frames = video_min_frames
+ # self.fps = video_fps
# Special tokens and IDs
- self.cls_token = self.CLS_TOKEN
- self.sep_token = self.SEP_TOKEN
- self.eos_token = self.EOS_TOKEN
- self.image_start = self.IMG_START
- self.image_end = self.IMG_END
- self.video_start = self.VID_START
- self.video_end = self.VID_END
- self.image_patch_id = self.tokenizer.convert_tokens_to_ids("<|IMAGE_PLACEHOLDER|>")
- self.image_start_id = self.tokenizer.convert_tokens_to_ids(self.image_start)
- self.video_start_id = self.tokenizer.convert_tokens_to_ids(self.video_start)
- self.sep_token_id = self.tokenizer.convert_tokens_to_ids(self.sep_token)
- self.eos_token_id = self.tokenizer.convert_tokens_to_ids(self.eos_token)
-
- self.token_type_mapping = self._build_token_type_mapping()
- self.is_training = True
+ # self.cls_token = "<|im_start|>"
+ # self.eos_token = "<|im_end|>"
+ self.vision_start = "<|vision_start|>"
+ self.vision_end = "<|vision_end|>"
+ self.image_token = "<|image_pad|>"
+ self.video_token = "<|video_pad|>"
+
+ self.image_token_id = self.tokenizer.convert_tokens_to_ids(self.image_token)
+ self.video_token_id = self.tokenizer.convert_tokens_to_ids(self.video_token)
+
+ self.vision_start_id = self.tokenizer.convert_tokens_to_ids(self.vision_start)
+ self.vision_start_id = self.tokenizer.convert_tokens_to_ids(self.vision_end)
+
self.role_prefixes = {
"system": "",
"user": "User: ",
@@ -144,26 +100,6 @@ def __init__(
"assistant": "Assistant: ",
}
- def _build_token_type_mapping(self) -> Dict[Any, int]:
- mapping = defaultdict(lambda: IDS_TYPE_FLAG["text"])
- for token in (
- self.IMG_START,
- self.IMG_END,
- self.VID_START,
- self.VID_END,
- ):
- mapping[token] = IDS_TYPE_FLAG["image"]
- mapping[self.image_patch_id] = IDS_TYPE_FLAG["image"]
- return mapping
-
- def train(self) -> None:
- """Enable training mode (produces labels)."""
- self.is_training = True
-
- def eval(self) -> None:
- """Enable evaluation mode (doesn't produce labels)."""
- self.is_training = False
-
def text2ids(self, text, images=None, videos=None):
"""
Convert chat text into model inputs.
@@ -251,48 +187,46 @@ def request2ids(
"video",
]:
image_message_list.append(item)
+ request["messages"] = messages
prompt_token_ids = self.apply_chat_template(request)
if len(prompt_token_ids) == 0:
raise ValueError("Invalid input: prompt_token_ids must be a non-empty sequence of token IDs")
- image_start_index = 0
- image_message_index = 0
+
+ vision_start_index = 0
+ vision_message_index = 0
for i in range(len(prompt_token_ids)):
- if prompt_token_ids[i] in [
- self.image_start_id,
- self.video_start_id,
- ]:
- self._add_text(prompt_token_ids[image_start_index : i + 1], outputs)
- image_start_index = i + 1
- image_message = image_message_list[image_message_index]
+ if prompt_token_ids[i] == self.vision_start_id :
+ self._add_text(prompt_token_ids[vision_start_index : i + 1], outputs)
+
+ vision_start_index = i + 1
+ image_message = image_message_list[vision_message_index]
+
if image_message["type"] == "image":
img = image_message.get("image")
if img is None:
continue
outputs["pic_cnt"] += 1
self._add_image(img, outputs)
+
elif image_message["type"] == "video":
video_bytes = image_message.get("video")
if video_bytes is None:
continue
frames = self._load_and_process_video(video_bytes, image_message)
+ # -----------
+ # mm_parser = MultiModalPartParser()
+ # fimg = mm_parser.parse_image("file:///home/liudongdong/github/FastDeploy/data/images/demo.jpeg")
+ # for i in range(len(frames)):
+ # frames[i] = fimg.copy()
+
outputs["video_cnt"] += 1
self._add_video(frames, outputs)
- image_message_index += 1
- self._add_text(prompt_token_ids[image_start_index:], outputs)
- if self.is_training:
- assert tgts, "training must give tgt !"
- self._extract_labels(outputs, tgts)
- return outputs
+ vision_message_index += 1
- def _add_special_token(self, token: Union[str, int], outputs: Dict) -> None:
- token_id = token if isinstance(token, int) else self.tokenizer.convert_tokens_to_ids(token)
- outputs["input_ids"].append(token_id)
- outputs["token_type_ids"].append(self.token_type_mapping[token])
- pos = outputs["cur_position"]
- outputs["position_ids"].append([pos] * 3)
- outputs["cur_position"] += 1
+ self._add_text(prompt_token_ids[vision_start_index:], outputs)
+ return outputs
def _add_text(self, tokens, outputs: Dict) -> None:
if isinstance(tokens, str):
@@ -314,7 +248,7 @@ def _add_image(self, img, outputs: Dict) -> None:
)[1]
num_tokens = (patches_h * patches_w) // (self.spatial_conv_size**2)
- outputs["input_ids"].extend([self.image_patch_id] * num_tokens)
+ outputs["input_ids"].extend([self.image_token_id] * num_tokens)
outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens)
pos_ids = self._compute_3d_positions(1, patches_h, patches_w, outputs["cur_position"])
@@ -322,14 +256,21 @@ def _add_image(self, img, outputs: Dict) -> None:
outputs["cur_position"] = np.max(pos_ids) + 1
# Preprocess pixels
+ # image_mean = [0.48145466, 0.4578275, 0.40821073]
+ # image_std = [0.26862954, 0.26130258, 0.27577711]
+ # do_rescale = True
+ # do_normalize = True
ret = self.image_preprocessor.preprocess(
images=[img.convert("RGB")],
- do_normalize=False,
- do_rescale=False,
+ # do_normalize=do_normalize,
+ # image_mean=image_mean,
+ # image_std=image_std,
+ # do_rescale=do_rescale,
predetermined_grid_thw=np.array([[patches_h, patches_w]]),
do_convert_rgb=True,
input_data_format=ChannelDimension.LAST,
)
+
outputs["images"].append(ret["pixel_values"])
outputs["grid_thw"].append(ret["image_grid_thw"])
outputs["image_type_ids"].append(0)
@@ -345,112 +286,48 @@ def _add_video(self, frames, outputs: Dict) -> None:
num_tokens = (num_frames * patches_h * patches_w) // (self.spatial_conv_size**2 * self.temporal_conv_size)
pixel_stack = np.stack([np.array(f.convert("RGB")) for f in frames], axis=0)
+
+ # rescale_factor = 1 / 255
+ # image_mean = [0.48145466, 0.4578275, 0.40821073]
+ # image_std = [0.26862954, 0.26130258, 0.27577711]
+ # do_rescale = True
+ # do_normalize = True
ret = self.image_preprocessor.preprocess(
images=None,
videos=pixel_stack,
- do_normalize=False,
- do_rescale=False,
+ # do_normalize=do_normalize,
+ # image_mean=image_mean,
+ # image_std=image_std,
+ # do_rescale=do_rescale,
predetermined_grid_thw=np.array([[patches_h, patches_w]] * num_frames),
do_convert_rgb=True,
input_data_format=ChannelDimension.LAST,
)
+
outputs["images"].append(ret["pixel_values_videos"])
outputs["grid_thw"].append(ret["video_grid_thw"])
+ # outputs["pixel_values_videos"].append(ret["pixel_values_videos"])
+ # outputs["video_grid_thw"].append(ret["video_grid_thw"])
outputs["image_type_ids"].extend([1] * num_frames)
- outputs["input_ids"].extend([self.image_patch_id] * num_tokens)
+ # num_tokens = ret["video_grid_thw"]
+ outputs["input_ids"].extend([self.video_token_id] * num_tokens)
outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens)
pos_ids = self._compute_3d_positions(num_frames, patches_h, patches_w, outputs["cur_position"])
outputs["position_ids"].extend(pos_ids)
outputs["cur_position"] = np.max(pos_ids) + 1
- def _extract_labels(self, outputs: Dict, tgts: List[str]) -> None:
- input_ids = copy.deepcopy(outputs["input_ids"])
- labels = [self.tokenizer.ignored_index] * len(input_ids)
-
- tgt_count = input_ids.count(self.sep_token_id)
- assert tgt_count == len(tgts), f"len(tgts) != len(src) {len(tgts)} vs {tgt_count}"
-
- tgt_index = 0
- for i, token_id in enumerate(input_ids):
- if token_id == self.sep_token_id:
- labels_token = self.tokenizer.tokenize(tgts[tgt_index])
- labels_token_id = self.tokenizer.convert_tokens_to_ids(labels_token)
- labels[i - len(labels_token_id) : i] = labels_token_id
- labels[i] = self.eos_token_id #
- tgt_index += 1
-
- outputs["labels"] = labels
-
def _load_and_process_video(self, url: str, item: Dict) -> List[Image.Image]:
- reader, meta, path = read_video_decord(url, save_to_disk=False)
-
- video_frame_args = dict()
- video_frame_args["fps"] = item.get("fps", self.fps)
- video_frame_args["min_frames"] = item.get("min_frames", self.min_frames)
- video_frame_args["max_frames"] = item.get("max_frames", self.max_frames)
- video_frame_args["target_frames"] = item.get("target_frames", self.target_frames)
- video_frame_args["frames_sample"] = item.get("frames_sample", self.frames_sample)
-
- video_frame_args = self._set_video_frame_args(video_frame_args, meta)
-
- frames_data, _, timestamps = read_frames_decord(
- path,
- reader,
- meta,
- target_frames=video_frame_args["target_frames"],
- target_fps=video_frame_args["fps"],
- frames_sample=video_frame_args["frames_sample"],
- save_to_disk=False,
- )
+ reader, meta = read_video_decord(url, save_to_disk=False)
- frames: List[Image.Image] = []
- for img_array, ts in zip(frames_data, timestamps):
- frames.append(render_frame_timestamp(img_array, ts))
- # Ensure even number of frames for temporal conv
- if len(frames) % 2 != 0:
- frames.append(copy.deepcopy(frames[-1]))
- return frames
+ frames = []
+ for i in range(meta["num_of_frame"]):
+ frame = reader[i].asnumpy()
+ image = Image.fromarray(frame, "RGB")
+ frames.append(image)
- def _set_video_frame_args(self, video_frame_args, video_meta):
- """
- 根据已知参数和优先级，设定最终的抽帧参数
- """
- # 优先级：video_target_frames > (video_min_frames, video_max_frames) > video_fps
- if video_frame_args["target_frames"] > 0:
- if video_frame_args["fps"] >= 0:
- raise ValueError("fps must be negative if target_frames is given")
- if (
- video_frame_args["min_frames"] > 0
- and video_frame_args["target_frames"] < video_frame_args["min_frames"]
- ):
- raise ValueError("target_frames must be larger than min_frames")
- if (
- video_frame_args["max_frames"] > 0
- and video_frame_args["target_frames"] > video_frame_args["max_frames"]
- ):
- raise ValueError("target_frames must be smaller than max_frames")
- else:
- if video_frame_args["fps"] < 0:
- raise ValueError("Must provide either positive target_fps or positive target_frames.")
- # 先计算在video_fps下抽到的帧数
- frames_to_extract = int(video_meta["duration"] * video_frame_args["fps"])
- # 判断是否在目标区间内，如果不是，则取target_frames为上界或下界
- if (
- video_frame_args["min_frames"] > 0
- and video_frame_args["max_frames"] > 0
- and video_frame_args["min_frames"] > video_frame_args["max_frames"]
- ):
- raise ValueError("min_frames must be smaller than max_frames")
- if video_frame_args["min_frames"] > 0 and frames_to_extract < video_frame_args["min_frames"]:
- video_frame_args["target_frames"] = video_frame_args["min_frames"]
- video_frame_args["fps"] = -1
- if video_frame_args["max_frames"] > 0 and frames_to_extract > video_frame_args["max_frames"]:
- video_frame_args["target_frames"] = video_frame_args["max_frames"]
- video_frame_args["fps"] = -1
-
- return video_frame_args
+ return frames
def _compute_3d_positions(self, t: int, h: int, w: int, start_idx: int) -> List[List[int]]:
# Downsample time if needed
@@ -470,16 +347,7 @@ def _load_tokenizer(self):
Returns:
tokenizer (AutoTokenizer)
"""
- vocab_file_names = [
- "tokenizer.model",
- "spm.model",
- "ernie_token_100k.model",
- ]
- for i in range(len(vocab_file_names)):
- if os.path.exists(os.path.join(self.model_name_or_path, vocab_file_names[i])):
- ErnieBotTokenizer.resource_files_names["vocab_file"] = vocab_file_names[i]
- break
- self.tokenizer = ErnieBotTokenizer.from_pretrained(self.model_name_or_path)
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path, padding_side="left", use_fast=True)
def apply_chat_template(self, request):
"""
@@ -495,18 +363,17 @@ def apply_chat_template(self, request):
if self.tokenizer.chat_template is None:
raise ValueError("This model does not support chat_template.")
- prompt_token_str = (
- self.tokenizer.apply_chat_template(
- request,
- tokenize=False,
- add_generation_prompt=request.get("add_generation_prompt", True),
- )
- .replace("<|image@placeholder|>", "")
- .replace("<|video@placeholder|>", "")
+ prompt_token_str = self.tokenizer.apply_chat_template(
+ request["messages"],
+ tokenize=False,
+ add_generation_prompt=request.get("add_generation_prompt", True),
)
+ prompt_token_str = prompt_token_str.replace(self.image_token, "").replace(self.video_token, "")
+
tokens = self.tokenizer.tokenize(prompt_token_str)
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
data_processor_logger.info(
f"req_id:{request.get('request_id', ''), } tokens: {tokens}, token_ids: {token_ids}"
)
return token_ids
+
diff --git a/fastdeploy/input/mm_processor/process_video.py b/fastdeploy/input/mm_processor/process_video.py
index 91120096c7..e82456bfcf 100644
--- a/fastdeploy/input/mm_processor/process_video.py
+++ b/fastdeploy/input/mm_processor/process_video.py
@@ -27,179 +27,18 @@
from .utils.video_utils import VideoReaderWrapper
-def read_video_decord(video_path, save_to_disk):
+def read_video_decord(video_path):
"""get reader and meta by decord"""
- # video_path = get_downloadable(video_path, save_to_disk=save_to_disk)
if isinstance(video_path, VideoReaderWrapper):
video_reader = video_path
else:
if isinstance(video_path, bytes):
video_path = io.BytesIO(video_path)
video_reader = VideoReaderWrapper(video_path, num_threads=1)
+
vlen = len(video_reader)
fps = video_reader.get_avg_fps()
duration = vlen / float(fps)
video_meta = {"fps": fps, "duration": duration, "num_of_frame": vlen}
-
- return video_reader, video_meta, video_path
-
-
-def get_frame_indices(
- vlen,
- target_frames=-1,
- target_fps=-1,
- frames_sample="middle",
- fix_start=None,
- input_fps=-1,
-):
- """
- 取出对应的frame index
- """
- assert frames_sample in ["rand", "middle", "leading"]
- if target_frames > 0:
- assert target_fps <= 0, "target_fps must be negative if target_frames is given."
- if target_frames > vlen:
- acc_samples = vlen
- data_processor_logger.info(
- f"target_frames={target_frames} is larger than video length {vlen}, "
- f"will sample {acc_samples} frames."
- )
- else:
- acc_samples = target_frames
- data_processor_logger.debug(f"sampling at target_frames={target_frames}, frames_sample={frames_sample}")
-
- # split the video into `acc_samples` intervals, and sample from each interval.
- intervals = np.linspace(start=0, stop=vlen, num=acc_samples + 1).astype(int)
- ranges = []
- for idx, interv in enumerate(intervals[:-1]):
- ranges.append((interv, intervals[idx + 1] - 1))
- if frames_sample == "rand":
- try:
- frame_indices = [random.choice(range(x[0], x[1])) for x in ranges]
- except Exception:
- frame_indices = np.random.permutation(vlen)[:acc_samples]
- frame_indices.sort()
- frame_indices = list(frame_indices)
- elif fix_start is not None:
- frame_indices = [x[0] + fix_start for x in ranges]
- elif frames_sample == "leading":
- frame_indices = [x[0] for x in ranges]
- elif frames_sample == "middle":
- frame_indices = [(x[0] + x[1]) // 2 for x in ranges]
- else:
- raise NotImplementedError
-
- elif target_fps > 0:
- assert target_frames <= 0, "target_frames must be negative if target_fps is given."
- assert input_fps > 0, "input_fps must be provided if target_fps is given."
- data_processor_logger.info(f"sampling at fps={target_fps}, frames_sample={frames_sample}")
- duration = float(vlen) / input_fps
- delta = 1 / target_fps # gap between frames, this is also the clip length each frame represents
- if frames_sample == "middle":
- frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta)
- elif frames_sample == "leading":
- frame_seconds = np.arange(0, duration, delta)
- if frames_sample == "rand":
- frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta)
- rand_offset = np.random.rand(*(frame_seconds.shape)) - 0.5
- frame_seconds += rand_offset * delta
- frame_indices = np.around(frame_seconds * input_fps).astype(int)
- frame_indices = [e for e in frame_indices if e < vlen]
-
- else:
- raise ValueError("Must provide either positive target_fps or positive target_frames.")
-
- return frame_indices
-
-
-def read_frames_decord(
- video_path,
- video_reader,
- video_meta,
- target_frames=-1,
- target_fps=-1,
- frames_sample="middle",
- fix_start=None,
- save_to_disk=False,
- cache_dir=EXTRACTED_FRAME_DIR,
- frame_indices=None,
- tol=10,
-):
- """get frames by decord"""
-
- if frame_indices is None:
- frame_indices = get_frame_indices(
- video_meta["num_of_frame"],
- target_frames=target_frames,
- target_fps=target_fps,
- frames_sample=frames_sample,
- fix_start=fix_start,
- input_fps=video_meta["fps"],
- )
-
- frames = []
- for frame_indice_index in range(0, len(frame_indices)):
- frame_indice = frame_indices[frame_indice_index]
- try:
- frames.append(video_reader[frame_indice].asnumpy()) # (T, H, W, C)
- except Exception as e:
- data_processor_logger.debug(f"encounter error when get frame: {frame_indice}, error: {e}")
- previous_counter = 1
- later_counter = 1
- previous_after_flag = True
- if frame_indice == 0 or frame_indice == len(video_reader) - 1:
- cur_tol = tol * 2
- else:
- cur_tol = tol
- while previous_counter < cur_tol or later_counter < cur_tol:
- if previous_after_flag:
- if frame_indice - previous_counter < 0:
- previous_counter += 1
- previous_after_flag = not previous_after_flag
- continue
- try:
- frames.append(video_reader[frame_indice - previous_counter].asnumpy())
- data_processor_logger.info(
- f"replace {frame_indice}-th frame with {frame_indice-previous_counter}-th frame"
- )
- frame_indices[frame_indice_index] = frame_indice - previous_counter
- break
- except Exception as e:
- previous_counter += 1
- data_processor_logger.info(f"error: {e}")
- else:
- if frame_indice + later_counter >= len(video_reader):
- later_counter += 1
- previous_after_flag = not previous_after_flag
- continue
- try:
- frames.append(video_reader[frame_indice + later_counter].asnumpy())
- data_processor_logger.info(
- f"replace {frame_indice}-th frame with {frame_indice+later_counter}-th frame"
- )
- frame_indices[frame_indice_index] = frame_indice + later_counter
- break
- except Exception:
- later_counter += 1
- previous_after_flag = not previous_after_flag
-
- frames = np.stack(frames, axis=0)
- assert len(frames) == len(frame_indices), f"len(frames): {len(frames)} != len(frame_indices): {len(frame_indices)}"
-
- ret = []
-
- url_sha1 = get_filename()
- for idx, frame in enumerate(frames):
- tmp = Image.fromarray(frame, "RGB")
- if save_to_disk:
- save_path = os.path.join(cache_dir, f"{url_sha1}", f"{idx}.png")
- if not os.path.exists(os.path.dirname(save_path)):
- os.makedirs(os.path.dirname(save_path))
- tmp.save(save_path)
- tmp = save_path
- ret.append(tmp)
-
- time_stamps = [frame_idx * video_meta["duration"] / video_meta["num_of_frame"] for frame_idx in frame_indices]
-
- return ret, frame_indices, time_stamps
+ return video_reader, video_meta
diff --git a/fastdeploy/input2/__init__.py b/fastdeploy/input2/__init__.py
new file mode 100644
index 0000000000..f4ede90624
--- /dev/null
+++ b/fastdeploy/input2/__init__.py
@@ -0,0 +1,15 @@
+"""
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
diff --git a/fastdeploy/input2/ernie_processor.py b/fastdeploy/input2/ernie_processor.py
new file mode 100644
index 0000000000..28d91bdbf8
--- /dev/null
+++ b/fastdeploy/input2/ernie_processor.py
@@ -0,0 +1,425 @@
+"""
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+import os
+
+import numpy as np
+from paddleformers.generation import GenerationConfig
+
+from fastdeploy.input.ernie_tokenizer import ErnieBotTokenizer
+from fastdeploy.input.text_processor import BaseDataProcessor
+from fastdeploy.utils import data_processor_logger
+
+_SAMPLING_EPS = 1e-5
+
+
+class ErnieProcessor(BaseDataProcessor):
+ """
+ 初始化模型实例。
+
+ Args:
+ model_name_or_path (str): 模型名称或路径。
+
+ Attributes:
+ model_name_or_path (str): 存储模型名称或路径。
+ decode_status (dict): 存储解码状态信息。
+ tokenizer (object): 存储分词器实例。
+ eos_token_ids (list): 存储结束符号的token ID列表。
+ eos_token_id_len (int): 存储结束符号的token ID列表的长度。
+ pad_token_id (int): 存储填充符号的token ID。
+ """
+
+ def __init__(self, model_name_or_path, reasoning_parser_obj=None):
+
+ self.model_name_or_path = model_name_or_path
+ data_processor_logger.info(f"model_name_or_path: {model_name_or_path}")
+
+ # Generation config
+ try:
+ self.generation_config = GenerationConfig.from_pretrained(self.model_name_or_path)
+ except Exception as e:
+ data_processor_logger.warning(
+ f"Can't find generation config, so it will not use "
+ f"generation_config field in the model config, details={e}"
+ )
+ self.generation_config = None
+
+ self.decode_status = dict()
+ self.thinking_parser_dict = dict()
+ self._load_tokenizer()
+ data_processor_logger.info(
+ f"tokenizer information: bos_token is {self.tokenizer.bos_token} \
+ {self.tokenizer.bos_token_id}, \
+ eos_token is {self.tokenizer.eos_token}, {self.tokenizer.eos_token_id} "
+ )
+ from paddleformers.trl.llm_utils import get_eos_token_id
+
+ self.eos_token_ids = get_eos_token_id(self.tokenizer, self.generation_config)
+ self.eos_token_id_len = len(self.eos_token_ids)
+ self.pad_token_id = self.get_pad_id()
+ self.reasoning_parser = None
+ if reasoning_parser_obj:
+ self.reasoning_parser = reasoning_parser_obj(self.tokenizer)
+
+ def process_request(self, request, max_model_len=None, **kwargs):
+ """
+ Preprocess the request
+
+ Args:
+ request (Dict): may contain text and messages fields
+
+ Returns:
+ bool: Whether preprocessing is successful
+ str: error message
+ """
+ request = self._apply_default_parameters(request)
+ if request.get("eos_token_ids") is None or len(request.eos_token_ids) == 0:
+ request.eos_token_ids = self.eos_token_ids
+ stop_sequences = request.get("stop", [])
+ if stop_sequences is not None and len(stop_sequences) != 0:
+ stop_seqs, stop_seqs_len = self.update_stop_seq(stop_sequences)
+ request.set("stop_token_ids", stop_seqs)
+ request.set("stop_seqs_len", stop_seqs_len)
+
+ if request.prompt_token_ids is None or len(request.prompt_token_ids) == 0:
+ if request.prompt is None and request.messages is None:
+ raise ValueError(f"The request should have `prompt_token_ids`, `prompt` or `messages`: {request}.")
+ if request.prompt is not None:
+ prompt = request.prompt if request.prompt is not None else request.messages[0]
+ prompt = prompt[0] if isinstance(prompt, list) else prompt
+ tokens = self.tokenizer.tokenize(prompt)
+ token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
+ request.prompt_token_ids = token_ids
+ data_processor_logger.info(f"req_id:{request.request_id}, tokens:{tokens}, token_ids: {token_ids}")
+ else:
+ request.prompt_token_ids = self.messages2ids(request.to_dict())
+
+ if len(request.prompt_token_ids) == 0:
+ raise ValueError("Invalid input: prompt_token_ids must be a non-empty sequence of token IDs")
+ if max_model_len is not None and len(request.prompt_token_ids) > max_model_len:
+ request.prompt_token_ids = request.prompt_token_ids[: max_model_len - 1]
+ if request.get("max_tokens") is None:
+ request.set(
+ "max_tokens",
+ max(1, max_model_len - len(request.prompt_token_ids)),
+ )
+ if request.get("temperature") < _SAMPLING_EPS:
+ # zero temperature is equivalent to greedy sampling
+ request.set("temperature", 1)
+ if request.get("top_p") < _SAMPLING_EPS:
+ request.set("top_p", _SAMPLING_EPS)
+ data_processor_logger.info(f"Processed request {request}")
+ return request
+
+ def process_request_dict(self, request, max_model_len=None):
+ """
+ Preprocess the request
+
+ Args:
+ request (Dict): may contain text and messages fields
+
+ Returns:
+ bool: Whether preprocessing is successful
+ str: error message
+ """
+ request = self._apply_default_parameters(request)
+ if not request.get("eos_token_ids"):
+ request["eos_token_ids"] = self.eos_token_ids
+
+ # processing stop_sequences
+ stop_sequences = request.get("stop", [])
+ if stop_sequences:
+ stop_seqs, stop_seqs_len = self.update_stop_seq(stop_sequences)
+ request["stop_token_ids"] = stop_seqs
+ request["stop_seqs_len"] = stop_seqs_len
+
+ # processing prompt_token_ids
+ if not request.get("prompt_token_ids"):
+ if request.get("prompt") is None and request.get("messages") is None:
+ raise ValueError(f"Request must contain 'prompt_token_ids', 'prompt', or 'messages': {request}")
+ if request.get("prompt"):
+ prompt = request.get("prompt")
+ prompt = prompt[0] if isinstance(prompt, list) else prompt
+
+ tokens = self.tokenizer.tokenize(prompt)
+ token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
+ request["prompt_token_ids"] = token_ids
+ req_id = request.get("request_id", None)
+ data_processor_logger.info(f"req_id:{req_id}, tokens:{tokens}, token_ids: {token_ids}")
+ else:
+ request["prompt_token_ids"] = self.messages2ids(request)
+ if len(request["prompt_token_ids"]) == 0:
+ raise ValueError("Invalid input: prompt_token_ids must be a non-empty sequence of token IDs")
+
+ # truncate prompts that exceed the length limit
+ if max_model_len is not None and len(request["prompt_token_ids"]) > max_model_len:
+ request["prompt_token_ids"] = request["prompt_token_ids"][: max_model_len - 1]
+ if request.get("max_tokens") is None:
+ request["max_tokens"] = max(1, max_model_len - len(request["prompt_token_ids"]))
+ if request.get("temperature") < _SAMPLING_EPS:
+ # zero temperature is equivalent to greedy sampling
+ request["temperature"] = 1
+ if request.get("top_p") < _SAMPLING_EPS:
+ request["top_p"] = _SAMPLING_EPS
+ data_processor_logger.info(f"Processed request {request}")
+
+ return request
+
+ def process_response(self, response_dict, **kwargs):
+ """
+ Preprocess the response
+
+ Args:
+ response_dict (Dict): response for engine, contain ids fields
+
+ Returns:
+ Dict: response contain text fields
+ """
+ req_id = response_dict.request_id
+ token_ids = response_dict.outputs.token_ids
+
+ response_dict.usage = {"completion_tokens": response_dict.outputs.index + 1}
+ if token_ids[-1] == self.tokenizer.eos_token_id:
+ token_ids = token_ids[:-1]
+ full_text = self.tokenizer.decode(token_ids)
+ if self.reasoning_parser:
+ reasoning_content, text = self.reasoning_parser.extract_reasoning_content(full_text, response_dict)
+ response_dict.outputs.text = text
+ response_dict.outputs.reasoning_content = reasoning_content
+ else:
+ response_dict.outputs.text = full_text
+ data_processor_logger.info(f"req_id:{req_id}, token)ids: {token_ids}")
+ if response_dict.outputs.text == "" and response_dict.outputs.reasoning_content == "":
+ return None
+ return response_dict
+
+ def process_response_dict(self, response_dict, stream, **kwargs):
+ """
+ Preprocess the response
+
+ Args:
+ response_dict (Dict): response for engine, contain ids fields
+
+ Returns:
+ Dict: response contain text fields
+ """
+ if stream:
+ return self.process_response_dict_streaming(response_dict, **kwargs)
+ else:
+ return self.process_response_dict_normal(response_dict, **kwargs)
+
+ def process_response_dict_normal(self, response_dict, **kwargs):
+ """
+ Preprocess the response
+
+ Args:
+ response_dict (Dict): response for engine, contain ids fields
+
+ Returns:
+ Dict: response contain text fields
+ """
+ enable_thinking = kwargs.get("enable_thinking")
+ token_ids = response_dict["outputs"]["token_ids"]
+ is_end = response_dict["finished"]
+ req_id = response_dict["request_id"]
+ if is_end and len(token_ids) > 0 and not kwargs.get("include_stop_str_in_output"):
+ if token_ids[-1] == self.tokenizer.eos_token_id:
+ token_ids = token_ids[:-1]
+ delta_text, _, previous_texts = self.ids2tokens(token_ids, req_id)
+ if is_end:
+ full_text = previous_texts + delta_text
+ if enable_thinking and self.reasoning_parser:
+ reasoning_content, text = self.reasoning_parser.extract_reasoning_content(full_text, response_dict)
+ response_dict["outputs"]["text"] = text
+ response_dict["outputs"]["reasoning_content"] = reasoning_content
+ else:
+ response_dict["outputs"]["text"] = full_text
+ data_processor_logger.info(f"req_id:{req_id}, decode_status: {self.decode_status[req_id]}")
+ del self.decode_status[req_id]
+ return response_dict
+
+ def process_response_dict_streaming(self, response_dict, **kwargs):
+ """
+ Preprocess the response streaming
+
+ Args:
+ response_dict (Dict): response for engine, contain ids fields
+
+ Returns:
+ Dict: response contain text fields
+ """
+ enable_thinking = kwargs.get("enable_thinking")
+ is_end = response_dict["finished"]
+ req_id = response_dict["request_id"]
+ token_ids = response_dict["outputs"]["token_ids"]
+
+ if is_end and len(token_ids) > 0 and not kwargs.get("include_stop_str_in_output"):
+ if token_ids[-1] == self.tokenizer.eos_token_id:
+ token_ids = token_ids[:-1]
+ delta_text, previous_token_ids, previous_texts = self.ids2tokens(token_ids, req_id)
+ if enable_thinking and self.reasoning_parser:
+ reasoning_content, text = self.reasoning_parser.extract_reasoning_content_streaming(
+ previous_texts,
+ previous_texts + delta_text,
+ delta_text,
+ previous_token_ids,
+ previous_token_ids + token_ids,
+ token_ids,
+ )
+ response_dict["outputs"]["text"] = text
+ response_dict["outputs"]["reasoning_content"] = reasoning_content
+ else:
+ response_dict["outputs"]["text"] = delta_text
+ if is_end:
+ data_processor_logger.info(f"req_id:{req_id}, decode_status: {self.decode_status[req_id]}")
+ del self.decode_status[req_id]
+ return response_dict
+
+ def messages2ids(self, request_or_messages):
+ """
+ Convert multi-turn messages into ID sequences.
+
+ Args:
+ request_or_messages: Either a request dict containing 'messages' field,
+ or a list of message dicts directly
+
+ Returns:
+ List of token IDs as strings (converted from token objects)
+ """
+ if self.tokenizer.chat_template is None:
+ raise ValueError("This model does not support chat_template.")
+ spliced_message = self.tokenizer.apply_chat_template(
+ request_or_messages,
+ tokenize=False,
+ split_special_tokens=False,
+ add_special_tokens=False,
+ )
+
+ req_id = None
+ if isinstance(request_or_messages, dict):
+ req_id = request_or_messages.get("request_id", None)
+ tokens = self.tokenizer.tokenize(spliced_message)
+ token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
+ data_processor_logger.info(f"req_id:{req_id}, tokens:{tokens}, token_ids: {token_ids}")
+ return token_ids
+
+ def ids2tokens(self, token_id, task_id):
+ """
+ token ids to strings
+
+ Args:
+ token_ids (List[int]): token ids
+ task_id (str): task id
+
+ Returns:
+ List[str]: strings
+ """
+
+ if task_id not in self.decode_status:
+ # prefix offset & read offset & history token ids & history token strings
+ self.decode_status[task_id] = [0, 0, [], ""]
+
+ prefix_offset = self.decode_status[task_id][0]
+ read_offset = self.decode_status[task_id][1]
+ previous_token_ids = self.decode_status[task_id][2]
+ previous_texts = self.decode_status[task_id][3]
+ decode_str, prefix_offset, read_offset = self.tokenizer.decode_token(
+ previous_token_ids + token_id, prefix_offset, read_offset
+ )
+ self.decode_status[task_id][0] = prefix_offset
+ self.decode_status[task_id][1] = read_offset
+ self.decode_status[task_id][2] += token_id
+ self.decode_status[task_id][3] += decode_str
+
+ return decode_str, previous_token_ids, previous_texts
+
+ def _load_tokenizer(self):
+ """
+ load tokenizer
+
+ Returns:
+ tokenizer (AutoTokenizer)
+ """
+ vocab_file_names = [
+ "tokenizer.model",
+ "spm.model",
+ "ernie_token_100k.model",
+ ]
+ for i in range(len(vocab_file_names)):
+ if os.path.exists(os.path.join(self.model_name_or_path, vocab_file_names[i])):
+ ErnieBotTokenizer.resource_files_names["vocab_file"] = vocab_file_names[i]
+ break
+ self.tokenizer = ErnieBotTokenizer.from_pretrained(self.model_name_or_path)
+
+ def get_pad_id(self):
+ """
+ get pad_token_id, if not pad_token_id, use eos_token
+
+ Returns:
+ int: pad_token_id
+ """
+ # if isinstance(self.tokenizer, (LlamaTokenizer, Llama3Tokenizer)) and not self.tokenizer.pad_token_id:
+ # return self.tokenizer.eos_token
+ return self.tokenizer.pad_token_id
+
+ def pad_batch_data(
+ self,
+ insts,
+ pad_id=0,
+ return_seq_len=False,
+ return_array=True,
+ pad_style="right",
+ ):
+ """Pad the instances to the max sequence length in batch."""
+ if len(insts) == 0:
+ padded_insts = np.array([[]], dtype=np.int64) if return_array else [[]]
+ if return_seq_len:
+ seq_len = np.array([], dtype=np.int64) if return_array else []
+ return padded_insts, seq_len
+ return padded_insts
+
+ max_len = max(map(len, insts))
+ if pad_style == "left":
+ padded_insts = [[pad_id] * (max_len - len(inst)) + list(inst) for inst in insts]
+ else:
+ padded_insts = [list(inst) + [pad_id] * (max_len - len(inst)) for inst in insts]
+ if return_array:
+ padded_insts = np.array(padded_insts, dtype=np.int64).reshape([-1, max_len])
+
+ if return_seq_len:
+ seq_len = [len(inst) for inst in insts]
+ if return_array:
+ seq_len = np.array(seq_len, dtype=np.int64).reshape(-1, 1)
+ return padded_insts, seq_len
+ return padded_insts
+
+ def update_stop_seq(self, stop_sequences):
+ """
+ Update stop sequences from request.
+ """
+ stop_seqs = []
+ if isinstance(stop_sequences, str):
+ stop_sequences = [stop_sequences]
+ for seq in stop_sequences:
+ if seq != self.tokenizer.eos_token_id:
+ stop_seqs.append(self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(seq)))
+ stop_seqs, stop_seqs_len = self.pad_batch_data(stop_seqs, pad_id=-1, return_seq_len=True, return_array=False)
+ data_processor_logger.debug(f"processed stop_seqs: {stop_seqs}, {stop_seqs_len}")
+ return stop_seqs, stop_seqs_len
+
+ def process_logprob_response(self, token_ids, **kwargs):
+ full_text = self.tokenizer.decode(token_ids, **kwargs)
+ return full_text
diff --git a/fastdeploy/input2/ernie_tokenizer.py b/fastdeploy/input2/ernie_tokenizer.py
new file mode 100644
index 0000000000..2bbc798c5c
--- /dev/null
+++ b/fastdeploy/input2/ernie_tokenizer.py
@@ -0,0 +1,394 @@
+"""
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+# cipher_token=WjI1fQOvhN # do not edit this line
+
+import os
+import re
+from shutil import copyfile
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+import paddle
+import sentencepiece as spm
+from paddleformers.transformers import PretrainedTokenizer
+from paddleformers.transformers.tokenizer_utils_base import PaddingStrategy, TextInput
+from paddleformers.utils.log import logger
+
+
+class ErnieBotTokenizer(PretrainedTokenizer):
+ """
+ 一个更好用的 `ErnieBotToknizer`，
+ 能 encode 目前 sft/ppo 阶段的特殊token，也支持多模态。
+ """
+
+ resource_files_names = {
+ "vocab_file": "tokenizer.model",
+ }
+ pretrained_resource_files_map = {"vocab_file": {"ernie-bot-10b": None}}
+ pretrained_init_configuration = {
+ "ernie-bot-10b": {},
+ }
+ model_input_names = [
+ "input_ids",
+ "position_ids",
+ "attention_mask",
+ "labels",
+ ]
+ padding_side = "right"
+
+ def __init__(
+ self,
+ vocab_file,
+ bos_token="",
+ cls_token="",
+ eos_token="",
+ mask_token="",
+ pad_token="",
+ sep_token="",
+ unk_token="",
+ additional_special_tokens=None,
+ verbose=False,
+ **kwargs,
+ ):
+ """doc"""
+ if additional_special_tokens is None:
+ additional_special_tokens = ["", ""]
+ super().__init__(
+ bos_token=bos_token,
+ cls_token=cls_token,
+ eos_token=eos_token,
+ mask_token=mask_token,
+ pad_token=pad_token,
+ sep_token=sep_token,
+ unk_token=unk_token,
+ additional_special_tokens=additional_special_tokens,
+ verbose=False,
+ **kwargs,
+ )
+ self.vocab_file = vocab_file
+ self.sp_model = spm.SentencePieceProcessor()
+ self.sp_model.Load(vocab_file)
+ # pre-process map-type all spec token for decode accelerate.
+
+ @property
+ def space_token(self):
+ """doc"""
+ return ""
+
+ @property
+ def space_token_id(self):
+ """doc"""
+ return self.sp_model.piece_to_id("")
+
+ @property
+ def gend_token(self):
+ """doc"""
+ return ""
+
+ @property
+ def gend_token_id(self):
+ """doc"""
+ return self.sp_model.piece_to_id("")
+
+ @property
+ def im_start_id(self):
+ """doc"""
+ return self.sp_model.piece_to_id("<|im_start|>")
+
+ @property
+ def im_end_id(self):
+ """doc"""
+ return self.sp_model.piece_to_id("<|im_end|>")
+
+ @property
+ def vocab_size(self):
+ """doc"""
+ return self.sp_model.vocab_size()
+
+ def get_vocab(self):
+ """doc"""
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+ vocab.update(self.added_tokens_encoder)
+ return vocab
+
+ def _tokenize(self, text):
+ """doc"""
+ return self.sp_model.encode_as_pieces(text)
+
+ def _convert_token_to_id(self, token):
+ """doc"""
+ return self.sp_model.piece_to_id(token)
+
+ def _convert_id_to_token(self, id):
+ """doc"""
+ return self.sp_model.id_to_piece(id)
+
+ def spec_init(self):
+ if not hasattr(self, "all_spec_tok"):
+ self.all_spec_tok = set(self.all_special_tokens)
+
+ def convert_tokens_to_string(self, tokens):
+ """Converts a sequence of tokens (string) in a single string."""
+ self.spec_init()
+ current_sub_tokens = []
+ out_string = ""
+ # prev_is_special = False
+ for token in tokens:
+ # make sure that special tokens are not decoded using sentencepiece model
+ if token in self.all_spec_tok:
+ # if not prev_is_special:
+ # out_string += " "
+ out_string += self.sp_model.decode(current_sub_tokens) + token
+ # prev_is_special = True
+
+ current_sub_tokens = []
+ else:
+ current_sub_tokens.append(token)
+ # prev_is_special = False
+ out_string += self.sp_model.decode(current_sub_tokens)
+ return out_string # .strip()
+
+ def prepare_for_model(self, *args, **kwargs):
+ """doc"""
+ if "add_special_tokens" in kwargs:
+ kwargs.pop("add_special_tokens")
+ # logger.warning(f'ErnieBotTokenizer v2 does not support `add_special_tokens`')
+ return super().prepare_for_model(*args, **kwargs)
+
+ def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
+ """
+ Save the vocabulary and special tokens file to a directory.
+ Args:
+ save_directory (`str`):
+ The directory in which to save the vocabulary.
+ Returns:
+ `Tuple(str)`: Paths to the files saved.
+ """
+ if not os.path.isdir(save_directory):
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+ return
+ out_vocab_file = os.path.join(
+ save_directory,
+ (filename_prefix + "-" if filename_prefix else "") + self.resource_files_names["vocab_file"],
+ )
+ if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
+ copyfile(self.vocab_file, out_vocab_file)
+ elif not os.path.isfile(self.vocab_file):
+ with open(out_vocab_file, "wb") as fi:
+ content_spiece_model = self.sp_model.serialized_model_proto()
+ fi.write(content_spiece_model)
+ return (out_vocab_file,)
+
+ def tokenize(self, text: TextInput, **kwargs) -> List[str]:
+ """
+ Converts a string in a sequence of tokens, using the tokenizer.
+
+ Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies
+ (BPE/SentencePieces/WordPieces). Takes care of added tokens.
+
+ Args:
+ text (`str`):
+ The sequence to be encoded.
+ **kwargs (additional keyword arguments):
+ Passed along to the model-specific `prepare_for_tokenization` preprocessing method.
+
+ Returns:
+ `List[str]`: The list of tokens.
+ """
+ # Simple mapping string => AddedToken for special tokens with specific tokenization behaviors
+ # all_special_tokens_extended = dict(
+ # (str(t), t)
+ # for t in self.all_special_tokens_extended
+ # if isinstance(t, AddedToken)
+ # )
+
+ self.spec_init()
+ text, kwargs = self.prepare_for_tokenization(text, **kwargs)
+
+ # TODO: should this be in the base class?
+ if hasattr(self, "do_lower_case") and self.do_lower_case:
+ # convert non-special tokens to lowercase
+ escaped_special_toks = [re.escape(s_tok) for s_tok in (self.unique_no_split_tokens + self.all_spec_tok)]
+ pattern = r"(" + r"|".join(escaped_special_toks) + r")|" + r"(.+?)"
+ text = re.sub(pattern, lambda m: m.groups()[0] or m.groups()[1].lower(), text)
+
+ no_split_token = set(self.unique_no_split_tokens)
+ tokens = self.tokens_trie.split(text)
+
+ # ["This is something", "", " else"]
+ # for i, token in enumerate(tokens):
+ # if token in no_split_token:
+ # tok_extended = all_special_tokens_extended.get(token, None)
+ # print(f'>>>{token}|{tok_extended}|{all_special_tokens_extended}<<<')
+ # left = tokens[i - 1] if i > 0 else None
+ # right = tokens[i + 1] if i < len(tokens) - 1 else None
+ # if isinstance(tok_extended, AddedToken):
+ # if tok_extended.rstrip and right:
+ # # A bit counter-intuitive but we strip the left of the string
+ # # since tok_extended.rstrip means the special token is eating all white spaces on its right
+ # tokens[i + 1] = right.lstrip()
+ # # Strip white spaces on the left
+ # if tok_extended.lstrip and left:
+ # tokens[i - 1] = left.rstrip() # Opposite here
+ # else:
+ # We strip left and right by default
+ # if right:
+ # tokens[i + 1] = right.lstrip()
+ # if left:
+ # tokens[i - 1] = left.rstrip()
+ # ["This is something", "", "else"]
+ tokenized_text = []
+ for token in tokens:
+ # Need to skip eventual empty (fully stripped) tokens
+ if not token:
+ continue
+ if token in no_split_token:
+ tokenized_text.append(token)
+ else:
+ tokenized_text.extend(self._tokenize(token))
+ # ["This", " is", " something", "", "else"]
+ return tokenized_text
+
+ def _decode(self, *args, **kwargs):
+ """doc"""
+ kwargs.pop("clean_up_tokenization_spaces", None)
+ kwargs.pop("spaces_between_special_tokens", None)
+ return super()._decode(
+ *args,
+ **kwargs,
+ clean_up_tokenization_spaces=False,
+ spaces_between_special_tokens=False,
+ )
+
+ def _pad(
+ self,
+ encoded_inputs: Dict,
+ max_length: Optional[int] = None,
+ padding_strategy=PaddingStrategy.DO_NOT_PAD,
+ pad_to_multiple_of: Optional[int] = None,
+ return_attention_mask: Optional[bool] = None,
+ ) -> dict:
+ """doc"""
+ if return_attention_mask is None:
+ return_attention_mask = "attention_mask" in self.model_input_names
+ if return_attention_mask:
+ required_input = encoded_inputs[self.model_input_names[0]]
+ if padding_strategy == PaddingStrategy.LONGEST:
+ max_length = len(required_input)
+ if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
+ max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
+ needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length
+ if "attention_mask" in encoded_inputs and encoded_inputs["attention_mask"] is not None:
+ attention_mask = encoded_inputs.pop("attention_mask")
+ if isinstance(attention_mask, paddle.Tensor):
+ attention_mask = attention_mask.numpy()
+ elif isinstance(attention_mask, list):
+ attention_mask = np.array(attention_mask)
+ elif not isinstance(attention_mask, np.ndarray):
+ raise ValueError(f"Unexpected type {type(attention_mask)} of attention_mask, ")
+ else:
+ attention_mask = np.tril(
+ np.ones(
+ (len(required_input), len(required_input)),
+ dtype=np.int64,
+ )
+ )
+ attention_mask = np.expand_dims(attention_mask, axis=0)
+ if needs_to_be_padded:
+ difference = max_length - len(required_input)
+ if self.padding_side == "right":
+ if attention_mask.ndim == 1:
+ pad_width = [(0, difference)]
+ else:
+ pad_width = [(0, 0), (0, difference), (0, difference)]
+ elif self.padding_side == "left":
+ if attention_mask.ndim == 1:
+ pad_width = [(difference, 0)]
+ else:
+ pad_width = [(0, 0), (difference, 0), (difference, 0)]
+ else:
+ raise ValueError("Invalid padding strategy:" + str(self.padding_side))
+ attention_mask = np.pad(
+ attention_mask,
+ pad_width=pad_width,
+ mode="constant",
+ constant_values=0,
+ )
+ encoded_inputs = super()._pad(
+ encoded_inputs,
+ max_length,
+ padding_strategy=padding_strategy,
+ pad_to_multiple_of=pad_to_multiple_of,
+ return_attention_mask=False,
+ )
+ if return_attention_mask:
+ encoded_inputs["attention_mask"] = attention_mask.tolist()
+ return encoded_inputs
+
+
+def add_special_tokens(
+ tokenizer,
+ special_tokens_info,
+ use_ocr_specialtoken=False,
+ use_crop_specialtoken=False,
+ special_token_ids_start=254208,
+ special_token_ids_end=256256,
+):
+ """
+ 增加 special token
+
+ placeholder [<|IMAGE_PLACEHOLDER|>, <|AUDIO_PLACEHOLDER|>, <|VIDEO_PLACEHOLDER|>] 共3个
+
+ 模态起始截止 special tokens [<|BOI|> <|EOI|> <|BOA|> <|EOA|> <|BOV|> <|EOV|>]
+
+ ocr special tokens [<|LOC_0|> <|LOC_1|> ... <|LOC_1000|>] 共1001个
+
+ crop special tokens [<|CROP_COL_SEP|>, <|CROP_ROW_SEP|>, <|CROP_IMAGE_SEP|>] 共3个
+ <|CROP_COL_SEP|> for col 维度切 图片width（替换原明文逗号）
+ <|CROP_ROW_SEP|> for row 维度切 图片height（替换原明文回车）
+ <|CROP_IMAGE_SEP|> for 区分原图和crop图 图片width（替换原明文两个回车）
+
+ 共2048个 unsed token
+
+ Args:
+ tokenizer (ErnieTokenizer): tokenizer
+ special_token_ids_start (int, optional): special token 起点 ids. Defaults to 254208.
+ special_token_ids_end (int, optional): 词表最多支持大小. Defaults to 256256.
+ """
+ special_tokens = [
+ special_tokens_info["image_placeholder"],
+ special_tokens_info["audio_placeholder"],
+ ]
+
+ if use_ocr_specialtoken:
+ special_tokens.extend(special_tokens_info["ocr_coor"])
+ special_tokens.extend(special_tokens_info["ocr_begin_end"])
+
+ if use_crop_specialtoken:
+ special_tokens.extend(special_tokens_info["crop"])
+
+ # add special_tokens
+ additional_special_tokens = {"additional_special_tokens": special_tokens}
+ tokenizer.add_special_tokens(additional_special_tokens)
+
+ # check
+ first_special_tokens = tokenizer.encode(special_tokens[0])["input_ids"]
+
+ assert first_special_tokens[0] == special_token_ids_start, f"[ERROR] first_special_tokens={first_special_tokens}"
+ assert (
+ len(tokenizer.get_vocab()) < special_token_ids_end
+ ), f"[ERROR] vocab_size = {len(tokenizer.get_vocab())} >= {special_token_ids_end} 增加过多special token了!"
diff --git a/fastdeploy/input2/ernie_vl_processor.py b/fastdeploy/input2/ernie_vl_processor.py
new file mode 100644
index 0000000000..63ae5bc310
--- /dev/null
+++ b/fastdeploy/input2/ernie_vl_processor.py
@@ -0,0 +1,285 @@
+"""
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+import numpy as np
+from paddleformers.generation import GenerationConfig
+
+from fastdeploy.engine.request import Request
+from fastdeploy.input.ernie_processor import ErnieProcessor
+from fastdeploy.input.mm_processor import IDS_TYPE_FLAG, DataProcessor
+from fastdeploy.utils import data_processor_logger
+
+
+class ErnieMoEVLProcessor(ErnieProcessor):
+ """The processor class for ERNIE MoE VL models."""
+
+ def __init__(
+ self,
+ model_name_or_path,
+ limit_mm_per_prompt=None,
+ mm_processor_kwargs=None,
+ reasoning_parser_obj=None,
+ ):
+ data_processor_logger.info(f"model_name_or_path: {model_name_or_path}")
+ tokenizer_path = model_name_or_path
+ preprocessor_path = model_name_or_path
+ processor_kwargs = self._parse_processor_kwargs(mm_processor_kwargs)
+
+ self.ernie_processor = DataProcessor(
+ tokenizer_name=tokenizer_path,
+ image_preprocessor_name=preprocessor_path,
+ **processor_kwargs,
+ )
+ self.ernie_processor.eval()
+ self.image_patch_id = self.ernie_processor.image_patch_id
+ self.spatial_conv_size = self.ernie_processor.spatial_conv_size
+
+ self.decode_status = dict()
+ self._load_tokenizer()
+
+ # Generation config
+ try:
+ self.generation_config = GenerationConfig.from_pretrained(model_name_or_path)
+ except Exception as e:
+ data_processor_logger.warning(
+ f"Can't find generation config: {e}, so it will not use generation_config field in the model config"
+ )
+ self.generation_config = None
+
+ # self.eos_token_ids = [self.tokenizer.eos_token_id]
+ from paddleformers.trl.llm_utils import get_eos_token_id
+
+ self.eos_token_ids = get_eos_token_id(self.tokenizer, self.generation_config)
+ self.eos_token_id_len = len(self.eos_token_ids)
+ self.pad_token_id = self.get_pad_id()
+ self.limit_mm_per_prompt = self._parse_limits(limit_mm_per_prompt)
+ self.reasoning_parser = None
+ if reasoning_parser_obj:
+ self.reasoning_parser = reasoning_parser_obj(self.tokenizer)
+
+ def get_pad_id(self):
+ """get pad id"""
+ return self.tokenizer.pad_token_id
+
+ def _load_tokenizer(self):
+ """
+ load tokenizer
+
+ Returns:
+ tokenizer (AutoTokenizer)
+ """
+ self.tokenizer = self.ernie_processor.tokenizer
+
+ def _apply_default_parameters(self, request):
+ """
+ Apply default value for parameters in request
+ """
+
+ def set_value(req, key, value):
+ value = getattr(self.generation_config, key, value)
+ if isinstance(req, dict):
+ if key not in req:
+ req[key] = value
+ else:
+ if req.get(key) is None:
+ req.set(key, value)
+
+ set_value(request, "top_p", 0.7)
+ set_value(request, "temperature", 1.0)
+ set_value(request, "repetition_penalty", 1.0)
+ set_value(request, "frequency_penalty", 0.0)
+ set_value(request, "presence_penalty", 0.0)
+ return request
+
+ def process_request(self, request, max_model_len=None, **kwargs):
+ """process the input data"""
+ task = request.to_dict()
+ task["enable_thinking"] = kwargs.get("enable_thinking", True)
+ self.process_request_dict(task, max_model_len)
+ request = Request.from_dict(task)
+ request = self._apply_default_parameters(request)
+
+ return request
+
+ def _parse_processor_kwargs(self, kwargs):
+ """解析多模态处理器参数配置"""
+ if not kwargs:
+ return {}
+
+ try:
+ if not isinstance(kwargs, dict):
+ raise ValueError("mm-processor-kwargs must be a dictionary")
+
+ # 验证参数类型
+ data_processor_logger.info(f"kwargs:{kwargs}")
+ expected_types = {
+ "spatial_conv_size": int,
+ "temporal_conv_size": int,
+ "image_min_pixels": int,
+ "image_max_pixels": int,
+ "video_min_pixels": int,
+ "video_max_pixels": int,
+ "video_target_frames": int,
+ "video_frames_sample": str,
+ "video_max_frames": int,
+ "video_min_frames": int,
+ "video_fps": int,
+ }
+
+ for key, value in kwargs.items():
+ if key in expected_types and not isinstance(value, expected_types[key]):
+ raise ValueError(
+ f"Invalid type for {key}: expected {expected_types[key].__name__}, got {type(value).__name__}"
+ )
+
+ return kwargs
+
+ except Exception as e:
+ data_processor_logger.warning(f"Invalid mm-processor-kwargs format: {e}")
+ return {}
+
+ def _parse_limits(self, limits):
+ """解析多模态限制配置"""
+ DEFAULT_LIMITS = {"image": 1, "video": 1, "audio": 1}
+
+ if not limits:
+ return DEFAULT_LIMITS
+
+ try:
+ if not isinstance(limits, dict):
+ raise ValueError("limit-mm-per-prompt must be a dictionary")
+ data_processor_logger.info(f"_parse_limits:{limits}")
+ return {**DEFAULT_LIMITS, **limits}
+ except Exception as e:
+ data_processor_logger.warning(f"Invalid limit-mm-per-prompt format: {e}, using default limits")
+ return DEFAULT_LIMITS
+
+ def _check_mm_limits(self, item):
+ if isinstance(item, dict):
+ # 请求包含prompt和multi_modal_data
+ mm_data = item
+ else:
+ # 请求包含messages
+ mm_data = {"image": [], "video": []}
+
+ for message in item:
+ if isinstance(message.get("content"), list):
+ for part in message["content"]:
+ if part.get("type") == "image":
+ mm_data["image"].append(part)
+ elif part.get("type") == "video":
+ mm_data["video"].append(part)
+
+ for modality, data in mm_data.items():
+ if modality in self.limit_mm_per_prompt:
+ limit = self.limit_mm_per_prompt[modality]
+ if len(data) > limit:
+ raise ValueError(f"Too many {modality} items in prompt, " f"got {len(data)} but limit is {limit}")
+
+ def process_request_dict(self, request, max_model_len=None):
+ """process the input data"""
+
+ request = self._apply_default_parameters(request)
+ if not request.get("eos_token_ids"):
+ request["eos_token_ids"] = self.eos_token_ids
+
+ stop_sequences = request.get("stop", [])
+ if stop_sequences:
+ stop_seqs, stop_seqs_len = self.update_stop_seq(stop_sequences)
+ request["stop_token_ids"] = stop_seqs
+ request["stop_seqs_len"] = stop_seqs_len
+
+ if request.get("prompt"):
+ multimodal_data = request.get("multimodal_data")
+ if multimodal_data is None:
+ multimodal_data = {}
+ self._check_mm_limits(multimodal_data)
+ images = multimodal_data.get("image", None)
+ videos = multimodal_data.get("video", None)
+ outputs = self.ernie_processor.text2ids(request["prompt"], images, videos)
+ elif request.get("messages"):
+ messages = request["messages"]
+ self._check_mm_limits(messages)
+ outputs = self.ernie_processor.request2ids(request)
+ else:
+ raise ValueError(f"Request must contain 'prompt', or 'messages': {request}")
+
+ metadata = request.get("metadata")
+ # 如果metadata包含之前输出的token，将这些token添加到input_ids末尾
+ if metadata and metadata.get("generated_token_ids"):
+ self.append_generated_tokens(outputs, metadata["generated_token_ids"])
+ outputs = self.pack_outputs(outputs)
+ request["prompt_token_ids"] = outputs["input_ids"].tolist()
+ request["prompt_token_ids_len"] = len(request["prompt_token_ids"])
+ request["multimodal_inputs"] = outputs
+
+ # 截断超过长度限制的prompt
+ if max_model_len is not None and len(request["prompt_token_ids"]) > max_model_len:
+ request["prompt_token_ids"] = request["prompt_token_ids"][: max_model_len - 1]
+ if request.get("max_tokens") is None:
+ request["max_tokens"] = max(1, max_model_len - len(request["prompt_token_ids"]))
+ data_processor_logger.info(f"Processed request {request}")
+
+ return request
+
+ def append_generated_tokens(self, multimodal_inputs, generated_token_ids):
+ "append already generated tokens"
+
+ num_tokens = len(generated_token_ids)
+ multimodal_inputs["input_ids"].extend(generated_token_ids)
+ multimodal_inputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * num_tokens)
+
+ start = multimodal_inputs["cur_position"]
+ for i in range(num_tokens):
+ multimodal_inputs["position_ids"].append([start + i] * 3)
+ multimodal_inputs["cur_position"] += num_tokens
+
+ def pack_outputs(self, outs):
+ # Stack or nullify image-related fields
+ if not outs["images"]:
+ outs["images"] = None
+ outs["grid_thw"] = None
+ outs["image_type_ids"] = None
+ else:
+ outs["images"] = np.vstack(outs["images"])
+ outs["grid_thw"] = np.vstack(outs["grid_thw"])
+ outs["image_type_ids"] = np.array(outs["image_type_ids"])
+
+ outs["image_patch_id"] = self.image_patch_id
+ # Convert lists to arrays
+ outs["input_ids"] = np.array(outs["input_ids"], dtype=np.int64)
+ outs["token_type_ids"] = np.array(outs["token_type_ids"], dtype=np.int64)
+ outs["position_ids"] = np.array(outs["position_ids"], dtype=np.int64)
+
+ return outs
+
+ def process_response_dict(self, response_dict, stream, **kwargs):
+ """
+ Preprocess the response
+
+ Args:
+ response_dict (Dict): response for engine, contain ids fields
+
+ Returns:
+ Dict: response contain text fields
+ """
+ enable_thinking = kwargs.pop("enable_thinking", True)
+ if enable_thinking is None:
+ enable_thinking = True
+ if stream:
+ return self.process_response_dict_streaming(response_dict, enable_thinking=enable_thinking, **kwargs)
+ else:
+ return self.process_response_dict_normal(response_dict, enable_thinking=enable_thinking, **kwargs)
diff --git a/fastdeploy/input2/mm_processor/__init__.py b/fastdeploy/input2/mm_processor/__init__.py
new file mode 100644
index 0000000000..ba59bc1654
--- /dev/null
+++ b/fastdeploy/input2/mm_processor/__init__.py
@@ -0,0 +1,23 @@
+"""
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+from .process import IDS_TYPE_FLAG, DataProcessor, fancy_print
+
+__all__ = [
+ "DataProcessor",
+ "fancy_print",
+ "IDS_TYPE_FLAG",
+]
diff --git a/fastdeploy/input2/mm_processor/image_preprocessor/__init__.py b/fastdeploy/input2/mm_processor/image_preprocessor/__init__.py
new file mode 100644
index 0000000000..c11444e675
--- /dev/null
+++ b/fastdeploy/input2/mm_processor/image_preprocessor/__init__.py
@@ -0,0 +1,20 @@
+"""
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+from .get_image_preprocessor import get_image_preprocessor
+from .image_preprocessor_adaptive import AdaptiveImageProcessor
+
+__all__ = ["get_image_preprocessor", "AdaptiveImageProcessor"]
diff --git a/fastdeploy/input2/mm_processor/image_preprocessor/get_image_preprocessor.py b/fastdeploy/input2/mm_processor/image_preprocessor/get_image_preprocessor.py
new file mode 100644
index 0000000000..0ff6f7d1ed
--- /dev/null
+++ b/fastdeploy/input2/mm_processor/image_preprocessor/get_image_preprocessor.py
@@ -0,0 +1,34 @@
+"""
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+"""get image preprocessor"""
+
+from fastdeploy.utils import data_processor_logger
+
+from .image_preprocessor_adaptive import AdaptiveImageProcessor
+
+
+def get_image_preprocessor(args):
+ """
+ get_image_preprocessor from args
+ """
+
+ if args.vision_model_name_or_path is None:
+ return None
+
+ data_processor_logger.info("use AdaptiveImageProcessor")
+ image_preprocess = AdaptiveImageProcessor.from_pretrained(args.vision_model_name_or_path)
+ return image_preprocess
diff --git a/fastdeploy/input2/mm_processor/image_preprocessor/image_preprocessor_adaptive.py b/fastdeploy/input2/mm_processor/image_preprocessor/image_preprocessor_adaptive.py
new file mode 100644
index 0000000000..15b15a4d22
--- /dev/null
+++ b/fastdeploy/input2/mm_processor/image_preprocessor/image_preprocessor_adaptive.py
@@ -0,0 +1,587 @@
+"""
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+"""image preprocessor adaptive"""
+
+import math
+from typing import List, Optional, Union
+
+import numpy as np
+import paddle
+import PIL
+from paddleformers.transformers.feature_extraction_utils import BatchFeature
+from paddleformers.transformers.image_processing_utils import BaseImageProcessor
+from paddleformers.transformers.image_transforms import (
+ convert_to_rgb,
+ normalize,
+ rescale,
+ resize,
+ to_channel_dimension_format,
+)
+from paddleformers.transformers.image_utils import (
+ ChannelDimension,
+ ImageInput,
+ PILImageResampling,
+ get_image_size,
+ infer_channel_dimension_format,
+ is_valid_image,
+ make_list_of_images,
+ to_numpy_array,
+ valid_images,
+)
+from paddleformers.transformers.tokenizer_utils_base import TensorType
+from PIL import Image
+
+from fastdeploy.utils import data_processor_logger
+
+OPENAI_CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073]
+OPENAI_CLIP_STD = [0.26862954, 0.26130258, 0.27577711]
+
+IMAGE_FACTOR = 28
+MIN_PIXELS = 4 * 28 * 28
+MAX_PIXELS = 16384 * 28 * 28
+MAX_RATIO = 200
+
+
+VideoInput = Union[
+ List["PIL.Image.Image"],
+ "np.ndarray",
+ "paddle.Tensor",
+ List["np.ndarray"],
+ List["paddle.Tensor"],
+ List[List["PIL.Image.Image"]],
+ List[List["np.ndarrray"]],
+ List[List["paddle.Tensor"]],
+]
+
+
+__all__ = [
+ "AdaptiveImageProcessor",
+]
+
+
+def is_scaled_image(image: np.ndarray) -> bool:
+ """
+ Checks to see whether the pixel values have already been rescaled to [0, 1].
+ """
+ if image.dtype == np.uint8:
+ return False
+
+ # It's possible the image has pixel values in [0, 255] but is of floating type
+ return np.min(image) >= 0 and np.max(image) <= 1
+
+
+def make_batched_images(images) -> List[List[ImageInput]]:
+ """
+ Accepts images in list or nested list format, and makes a list of images for preprocessing.
+
+ Args:
+ images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`):
+ The input image.
+
+ Returns:
+ list: A list of images.
+ """
+ if isinstance(images, (list, tuple)) and isinstance(images[0], (list, tuple)) and is_valid_image(images[0][0]):
+ return [img for img_list in images for img in img_list]
+
+ elif isinstance(images, (list, tuple)) and is_valid_image(images[0]):
+ return images
+
+ elif is_valid_image(images):
+ return [images]
+
+ raise ValueError(f"Could not make batched images from {images}")
+
+
+# Copied from transformers.models.llava_next_video.image_processing_llava_next_video.make_batched_videos
+def make_batched_videos(videos) -> List[VideoInput]:
+ """dummy"""
+ if isinstance(videos, (list, tuple)) and isinstance(videos[0], (list, tuple)) and is_valid_image(videos[0][0]):
+ return videos
+
+ elif isinstance(videos, (list, tuple)) and is_valid_image(videos[0]):
+ if isinstance(videos[0], Image.Image):
+ return [videos]
+ elif len(videos[0].shape) == 4:
+ return [list(video) for video in videos]
+
+ elif is_valid_image(videos) and len(videos.shape) == 4:
+ return [list(videos)]
+
+ raise ValueError(f"Could not make batched video from {videos}")
+
+
+class AdaptiveImageProcessor(BaseImageProcessor):
+ r"""
+ Constructs a adaptive image processor that dynamically resizes images based on the original images.
+
+ Args:
+ do_resize (`bool`, *optional*, defaults to `True`):
+ Whether to resize the image's (height, width) dimensions.
+ resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
+ Resampling filter to use when resizing the image.
+ do_rescale (`bool`, *optional*, defaults to `True`):
+ Whether to rescale the image by the specified scale `rescale_factor`.
+ rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
+ Scale factor to use if rescaling the image.
+ do_normalize (`bool`, *optional*, defaults to `True`):
+ Whether to normalize the image.
+ image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
+ Mean to use if normalizing the image. This is a float or list of floats for each channel in the image.
+ image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
+ Standard deviation to use if normalizing the image. This is a float or list of floats for each channel
+ in the image.
+ do_convert_rgb (`bool`, *optional*, defaults to `True`):
+ Whether to convert the image to RGB.
+ min_pixels (`int`, *optional*, defaults to `56 * 56`):
+ The min pixels of the image to resize the image.
+ max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`):
+ The max pixels of the image to resize the image.
+ patch_size (`int`, *optional*, defaults to 14):
+ The spacial patch size of the vision encoder.
+ temporal_conv_size (`int`, *optional*, defaults to 2):
+ The temporal conv size in resampler.
+ merge_size (`int`, *optional*, defaults to 2):
+ The merge size of the vision encoder to llm encoder.
+ """
+
+ model_input_names = [
+ "pixel_values",
+ "image_grid_thw",
+ "pixel_values_videos",
+ "video_grid_thw",
+ ]
+
+ def __init__(
+ self,
+ do_resize: bool = True,
+ resample: PILImageResampling = PILImageResampling.BICUBIC,
+ do_rescale: bool = True,
+ rescale_factor: float = 1 / 255,
+ do_normalize: bool = True,
+ image_mean: Optional[Union[float, List[float]]] = None,
+ image_std: Optional[Union[float, List[float]]] = None,
+ do_convert_rgb: bool = True,
+ min_pixels: int = 56 * 56,
+ max_pixels: int = 28 * 28 * 1280,
+ patch_size: int = 14,
+ temporal_conv_size: int = 2,
+ merge_size: int = 2,
+ **kwargs,
+ ) -> None:
+ """init"""
+ super().__init__(**kwargs)
+ self.do_resize = do_resize
+ self.resample = resample
+ self.do_rescale = do_rescale
+ self.rescale_factor = rescale_factor
+ self.do_normalize = do_normalize
+ self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
+ self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
+ self.min_pixels = min_pixels
+ self.max_pixels = max_pixels
+ self.patch_size = patch_size
+ self.temporal_conv_size = temporal_conv_size
+ self.merge_size = merge_size
+ self.size = {"min_pixels": min_pixels, "max_pixels": max_pixels}
+ self.do_convert_rgb = do_convert_rgb
+
+ def set_pixels(self, min_pixels=None, max_pixels=None, msg=""):
+ """设定pixels"""
+ if min_pixels is not None:
+ assert isinstance(min_pixels, int) and min_pixels >= 0, "min_pixels must be positive int"
+ data_processor_logger.info(f"{msg} AdaptiveImageProcessor set min_pixels = {min_pixels}")
+ self.min_pixels = min_pixels
+ self.size["min_pixels"] = int(min_pixels)
+ if max_pixels is not None:
+ assert isinstance(max_pixels, int) and max_pixels > 0, "max_pixels must be positive int"
+ data_processor_logger.info(f"{msg} AdaptiveImageProcessor set max_pixels = {max_pixels}")
+ self.max_pixels = max_pixels
+ self.size["max_pixels"] = int(max_pixels)
+
+ def get_smarted_resize(self, height, width, min_pixels=None, max_pixels=None):
+ """dummy"""
+ actual_min_pixels = min_pixels if min_pixels is not None else self.min_pixels
+ actual_max_pixels = max_pixels if max_pixels is not None else self.max_pixels
+ resized_height, resized_width = smart_resize(
+ height,
+ width,
+ factor=self.patch_size * self.merge_size,
+ min_pixels=actual_min_pixels,
+ max_pixels=actual_max_pixels,
+ )
+ return (resized_height, resized_width), (
+ resized_height // self.patch_size,
+ resized_width // self.patch_size,
+ )
+
+ def _preprocess(
+ self,
+ images: Union[ImageInput, VideoInput],
+ do_resize: bool = True,
+ resample: PILImageResampling = None,
+ do_rescale: bool = True,
+ rescale_factor: float = 1 / 255,
+ do_normalize: bool = True,
+ image_mean: Optional[Union[float, List[float]]] = None,
+ image_std: Optional[Union[float, List[float]]] = None,
+ do_convert_rgb: bool = False,
+ data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+ input_data_format: Optional[Union[str, ChannelDimension]] = None,
+ predetermined_grid_thw=None,
+ ):
+ """
+ Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.
+
+ Args:
+ images (`ImageInput`):
+ Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255.
+ If pixel values range from 0 to 1, set `do_rescale=False`.
+ vision_info (`List[Dict]`, *optional*):
+ Optional list of dictionaries containing additional information about vision inputs.
+ do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+ Whether to resize the image.
+ resample (`PILImageResampling`, *optional*, defaults to `self.resample`):
+ Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums.
+ do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+ Whether to rescale the image.
+ rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+ Scale factor to use if rescaling the image.
+ do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+ Whether to normalize the image.
+ image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+ Mean to use if normalizing the image.
+ Can be a float or a list of floats corresponding to the number of channels in the image.
+ image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+ Standard deviation to use if normalizing the image.
+ Can be a float or a list of floats corresponding to the number of channels in the image.
+ do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+ Whether to convert the image to RGB.
+ data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`):
+ The channel dimension format for the output image. Can be one of:
+ - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+ - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+ - Unset: Use the channel dimension format of the input image.
+ input_data_format (`ChannelDimension` or `str`, *optional*):
+ The channel dimension format for the input image. Can be one of:
+ - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+ - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+ - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+ - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+ """
+ images = make_list_of_images(images)
+
+ if do_convert_rgb:
+ images = [convert_to_rgb(image) for image in images]
+
+ # All transformations expect numpy arrays.
+ images = [to_numpy_array(image) for image in images]
+
+ if is_scaled_image(images[0]) and do_rescale:
+ data_processor_logger.warning(
+ "It looks like you are trying to rescale already rescaled images. If the input"
+ " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
+ )
+ if input_data_format is None:
+ # We assume that all images have the same channel dimension format.
+ input_data_format = infer_channel_dimension_format(images[0])
+
+ height, width = get_image_size(images[0], channel_dim=input_data_format)
+ resized_height, resized_width = height, width
+ processed_images = []
+
+ if predetermined_grid_thw is not None:
+ assert len(predetermined_grid_thw) == len(
+ images
+ ), f"len(predetermined_grid_thw) {len(predetermined_grid_thw)} == len(images) {len(images)}"
+
+ for img_idx, image in enumerate(images):
+ if do_resize:
+ if predetermined_grid_thw is not None:
+ (resized_height, resized_width) = predetermined_grid_thw[img_idx]
+ resized_height *= self.patch_size
+ resized_width *= self.patch_size
+ else:
+ resized_height, resized_width = smart_resize(
+ height,
+ width,
+ factor=self.patch_size * self.merge_size,
+ min_pixels=self.min_pixels,
+ max_pixels=self.max_pixels,
+ )
+ image = image.astype("uint8") # TODO : 需要手动加上，否则多除255 导致结果会出错
+ # 直接fromarray，不要靠paddleformers里面的
+ image = Image.fromarray(image)
+ image = resize(
+ image,
+ size=(resized_height, resized_width),
+ resample=resample,
+ data_format=input_data_format,
+ )
+ if do_rescale:
+ image = rescale(image, scale=rescale_factor, data_format=input_data_format)
+
+ if do_normalize:
+ image = normalize(
+ image=image,
+ mean=image_mean,
+ std=image_std,
+ data_format=input_data_format,
+ )
+
+ image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) # [C, H, W]
+
+ processed_images.append(image)
+ patches = np.array(processed_images)
+ if data_format == ChannelDimension.LAST:
+ patches = patches.transpose([0, 3, 1, 2])
+
+ channel = patches.shape[1] # [time, C, H, W]
+ grid_t = patches.shape[0]
+ grid_h, grid_w = (
+ resized_height // self.patch_size,
+ resized_width // self.patch_size,
+ )
+ patches = patches.reshape(
+ [
+ grid_t,
+ channel,
+ grid_h // self.merge_size,
+ self.merge_size,
+ self.patch_size,
+ grid_w // self.merge_size,
+ self.merge_size,
+ self.patch_size,
+ ]
+ )
+ # [grid_t, grid_h/merge_size, grid_w/merge_size, merge_size, merge_size, C, psz, psz]
+ patches = patches.transpose([0, 2, 5, 3, 6, 1, 4, 7])
+
+ flatten_patches = patches.reshape(
+ [
+ grid_t * grid_h * grid_w,
+ channel * self.patch_size * self.patch_size,
+ ]
+ ) # [grid_t * grid_h * grid_w, C * psz * psz]
+
+ return flatten_patches, (grid_t, grid_h, grid_w)
+
+ def preprocess(
+ self,
+ images: ImageInput,
+ videos: VideoInput = None,
+ do_resize: bool = True,
+ size: Optional[Union[int, List[int]]] = None,
+ resample: PILImageResampling = None,
+ do_rescale: bool = True,
+ rescale_factor: float = 1 / 255,
+ do_normalize: bool = True,
+ image_mean: Optional[Union[float, List[float]]] = None,
+ image_std: Optional[Union[float, List[float]]] = None,
+ do_convert_rgb: bool = False,
+ return_tensors: Optional[Union[str, TensorType]] = None,
+ data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+ input_data_format: Optional[Union[str, ChannelDimension]] = None,
+ predetermined_grid_thw=None,
+ ):
+ """
+ Args:
+ images (`ImageInput`):
+ Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
+ passing in images with pixel values between 0 and 1, set `do_rescale=False`.
+ videos (`VideoInput`):
+ Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If
+ passing in videos with pixel values between 0 and 1, set `do_rescale=False`.
+ do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+ Whether to resize the image.
+ size (`Dict[str, int]`, *optional*, defaults to `self.size`):
+ Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
+ the longest edge resized to keep the input aspect ratio.
+ resample (`int`, *optional*, defaults to `self.resample`):
+ Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
+ has an effect if `do_resize` is set to `True`.
+ do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+ Whether to rescale the image.
+ rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+ Rescale factor to rescale the image by if `do_rescale` is set to `True`.
+ do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+ Whether to normalize the image.
+ image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+ Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+ image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+ Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+ `True`.
+ do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+ Whether to convert the image to RGB.
+ return_tensors (`str` or `TensorType`, *optional*):
+ The type of tensors to return. Can be one of:
+ - Unset: Return a list of `np.ndarray`.
+ - `TensorType.PADDLE` or `'pt'`: Return a batch of type `torch.Tensor`.
+ - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
+ data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
+ The channel dimension format for the output image. Can be one of:
+ - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+ - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+ - Unset: Use the channel dimension format of the input image.
+ input_data_format (`ChannelDimension` or `str`, *optional*):
+ The channel dimension format for the input image. If unset, the channel dimension format is inferred
+ from the input image. Can be one of:
+ - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+ - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+ - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+
+ """
+ do_resize = do_resize if do_resize is not None else self.do_resize
+ size = size if size is not None else self.size
+ resample = resample if resample is not None else self.resample
+ do_rescale = do_rescale if do_rescale is not None else self.do_rescale
+ rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor
+ do_normalize = do_normalize if do_normalize is not None else self.do_normalize
+ image_mean = image_mean if image_mean is not None else self.image_mean
+ image_std = image_std if image_std is not None else self.image_std
+ do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
+
+ if images is not None:
+ images = make_batched_images(images)
+ if videos is not None:
+ videos = make_batched_videos(videos)
+
+ if images is not None and not valid_images(images):
+ raise ValueError("Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " "paddle.Tensor.")
+
+ if images is not None:
+ pixel_values, vision_grid_thws = [], []
+ for img_idx, image in enumerate(images):
+ if predetermined_grid_thw is not None:
+ predetermined_grid_thw_one = [predetermined_grid_thw[img_idx]]
+ else:
+ predetermined_grid_thw_one = None
+ patches, image_grid_thw = self._preprocess(
+ image,
+ do_resize=do_resize,
+ resample=resample,
+ do_rescale=do_rescale,
+ rescale_factor=rescale_factor,
+ do_normalize=do_normalize,
+ image_mean=image_mean,
+ image_std=image_std,
+ data_format=data_format,
+ do_convert_rgb=do_convert_rgb,
+ input_data_format=input_data_format,
+ predetermined_grid_thw=predetermined_grid_thw_one,
+ )
+ pixel_values.extend(patches)
+ vision_grid_thws.append(image_grid_thw)
+ pixel_values = np.array(pixel_values)
+ vision_grid_thws = np.array(vision_grid_thws)
+ data = {
+ "pixel_values": pixel_values,
+ "image_grid_thw": vision_grid_thws,
+ }
+
+ if videos is not None:
+ pixel_values, vision_grid_thws = [], []
+ for images in videos:
+ patches, video_grid_thw = self._preprocess(
+ images,
+ do_resize=do_resize,
+ resample=resample,
+ do_rescale=do_rescale,
+ rescale_factor=rescale_factor,
+ do_normalize=do_normalize,
+ image_mean=image_mean,
+ image_std=image_std,
+ data_format=data_format,
+ do_convert_rgb=do_convert_rgb,
+ input_data_format=input_data_format,
+ predetermined_grid_thw=predetermined_grid_thw,
+ )
+ pixel_values.extend(patches)
+ vision_grid_thws.append(video_grid_thw)
+ pixel_values = np.array(pixel_values)
+ vision_grid_thws = np.array(vision_grid_thws)
+
+ data = {
+ "pixel_values_videos": pixel_values,
+ "video_grid_thw": vision_grid_thws,
+ }
+
+ return BatchFeature(data=data, tensor_type=return_tensors)
+
+
+def round_by_factor(number: int, factor: int) -> int:
+ """Returns the closest integer to 'number' that is divisible by 'factor'."""
+ return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+ """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+ return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+ """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+ return math.floor(number / factor) * factor
+
+
+def smart_resize(
+ height: int,
+ width: int,
+ factor: int = IMAGE_FACTOR,
+ min_pixels: int = MIN_PIXELS,
+ max_pixels: int = MAX_PIXELS,
+):
+ """
+ Rescales the image so that the following conditions are met:
+
+ 1. Both dimensions (height and width) are divisible by 'factor'.
+
+ 2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+ 3. The aspect ratio of the image is maintained as closely as possible.
+ """
+ if max(height, width) / min(height, width) > MAX_RATIO:
+ if height > width:
+ new_width = max(factor, round_by_factor(width, factor))
+ new_height = floor_by_factor(new_width * MAX_RATIO, factor)
+ else:
+ new_height = max(factor, round_by_factor(height, factor))
+ new_width = floor_by_factor(new_height * MAX_RATIO, factor)
+
+ data_processor_logger.info(
+ f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)},\
+ resize to {max(new_height, new_width) / min(new_height, new_width)}"
+ )
+
+ height = new_height
+ width = new_width
+
+ h_bar = max(factor, round_by_factor(height, factor))
+ w_bar = max(factor, round_by_factor(width, factor))
+ if h_bar * w_bar > max_pixels:
+ beta = math.sqrt((height * width) / max_pixels)
+ h_bar = floor_by_factor(height / beta, factor)
+ w_bar = floor_by_factor(width / beta, factor)
+ elif h_bar * w_bar < min_pixels:
+ beta = math.sqrt(min_pixels / (height * width))
+ h_bar = ceil_by_factor(height * beta, factor)
+ w_bar = ceil_by_factor(width * beta, factor)
+
+ if min_pixels > h_bar * w_bar or h_bar * w_bar > max_pixels:
+ raise ValueError(f"encounter invalid h_bar: {h_bar}, w_bar: {w_bar}")
+
+ return h_bar, w_bar
diff --git a/fastdeploy/input2/mm_processor/process.py b/fastdeploy/input2/mm_processor/process.py
new file mode 100644
index 0000000000..ea2559a0fe
--- /dev/null
+++ b/fastdeploy/input2/mm_processor/process.py
@@ -0,0 +1,512 @@
+"""
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+""" process.py """
+import copy
+import os
+from collections import defaultdict
+from typing import Any, Dict, List, Union
+
+import numpy as np
+from paddleformers.transformers.image_utils import ChannelDimension
+from PIL import Image
+
+from fastdeploy.entrypoints.chat_utils import parse_chat_messages
+from fastdeploy.input.ernie_tokenizer import ErnieBotTokenizer
+from fastdeploy.utils import data_processor_logger
+
+from .image_preprocessor.image_preprocessor_adaptive import AdaptiveImageProcessor
+from .process_video import read_frames_decord, read_video_decord
+from .utils.render_timestamp import render_frame_timestamp
+
+IDS_TYPE_FLAG = {"text": 0, "image": 1, "video": 2, "audio": 3}
+
+
+def fancy_print(input_ids, tokenizer, image_patch_id=None):
+ """
+ input_ids: input_ids
+ tokenizer: the tokenizer of models
+ """
+ i = 0
+ res = ""
+ text_ids = []
+ real_image_token_len = 0
+ while i < len(input_ids):
+ if input_ids[i] == image_patch_id:
+ if len(text_ids) > 0:
+ res += tokenizer.decode(text_ids)
+ text_ids = []
+
+ real_image_token_len += 1
+ else:
+ if real_image_token_len != 0:
+ res += f"<|IMAGE@{real_image_token_len}|>"
+ real_image_token_len = 0
+
+ text_ids.append(input_ids[i])
+
+ i += 1
+ if len(text_ids) > 0:
+
+ res += tokenizer.decode(text_ids)
+ text_ids = []
+ return res
+
+
+class DataProcessor:
+ """
+ Processes multimodal chat messages into model-ready inputs,
+ handling text, images, and videos with 3D positional embeddings.
+ """
+
+ CLS_TOKEN = "<|begin_of_sentence|>"
+ SEP_TOKEN = "<|end_of_sentence|>"
+ EOS_TOKEN = ""
+ IMG_START = "<|IMAGE_START|>"
+ IMG_END = "<|IMAGE_END|>"
+ VID_START = "<|VIDEO_START|>"
+ VID_END = "<|VIDEO_END|>"
+
+ def __init__(
+ self,
+ tokenizer_name: str,
+ image_preprocessor_name: str,
+ spatial_conv_size: int = 2,
+ temporal_conv_size: int = 2,
+ image_min_pixels: int = 4 * 28 * 28,
+ image_max_pixels: int = 6177 * 28 * 28,
+ video_min_pixels: int = 299 * 28 * 28,
+ video_max_pixels: int = 1196 * 28 * 28,
+ video_target_frames: int = -1,
+ video_frames_sample: str = "leading",
+ video_max_frames: int = 180,
+ video_min_frames: int = 16,
+ video_fps: int = 2,
+ **kwargs,
+ ) -> None:
+ # Tokenizer and image preprocessor
+ self.model_name_or_path = tokenizer_name
+ self._load_tokenizer()
+ self.tokenizer.ignored_index = -100
+ self.image_preprocessor = AdaptiveImageProcessor.from_pretrained(image_preprocessor_name)
+
+ # Convolution sizes for patch aggregation
+ self.spatial_conv_size = spatial_conv_size
+ self.temporal_conv_size = temporal_conv_size
+
+ # Pixel constraints
+ self.image_min_pixels = image_min_pixels
+ self.image_max_pixels = image_max_pixels
+ self.video_min_pixels = video_min_pixels
+ self.video_max_pixels = video_max_pixels
+
+ # Video sampling parameters
+ self.target_frames = video_target_frames
+ self.frames_sample = video_frames_sample
+ self.max_frames = video_max_frames
+ self.min_frames = video_min_frames
+ self.fps = video_fps
+
+ # Special tokens and IDs
+ self.cls_token = self.CLS_TOKEN
+ self.sep_token = self.SEP_TOKEN
+ self.eos_token = self.EOS_TOKEN
+ self.image_start = self.IMG_START
+ self.image_end = self.IMG_END
+ self.video_start = self.VID_START
+ self.video_end = self.VID_END
+ self.image_patch_id = self.tokenizer.convert_tokens_to_ids("<|IMAGE_PLACEHOLDER|>")
+ self.image_start_id = self.tokenizer.convert_tokens_to_ids(self.image_start)
+ self.video_start_id = self.tokenizer.convert_tokens_to_ids(self.video_start)
+ self.sep_token_id = self.tokenizer.convert_tokens_to_ids(self.sep_token)
+ self.eos_token_id = self.tokenizer.convert_tokens_to_ids(self.eos_token)
+
+ self.token_type_mapping = self._build_token_type_mapping()
+ self.is_training = True
+ self.role_prefixes = {
+ "system": "",
+ "user": "User: ",
+ "bot": "Assistant: ",
+ "assistant": "Assistant: ",
+ }
+
+ def _build_token_type_mapping(self) -> Dict[Any, int]:
+ mapping = defaultdict(lambda: IDS_TYPE_FLAG["text"])
+ for token in (
+ self.IMG_START,
+ self.IMG_END,
+ self.VID_START,
+ self.VID_END,
+ ):
+ mapping[token] = IDS_TYPE_FLAG["image"]
+ mapping[self.image_patch_id] = IDS_TYPE_FLAG["image"]
+ return mapping
+
+ def train(self) -> None:
+ """Enable training mode (produces labels)."""
+ self.is_training = True
+
+ def eval(self) -> None:
+ """Enable evaluation mode (doesn't produce labels)."""
+ self.is_training = False
+
+ def text2ids(self, text, images=None, videos=None):
+ """
+ Convert chat text into model inputs.
+ Returns a dict with input_ids, token_type_ids, position_ids, images, grid_thw, image_type_ids, labels.
+ """
+
+ outputs = {
+ "input_ids": [],
+ "token_type_ids": [],
+ "position_ids": [],
+ "images": [],
+ "grid_thw": [],
+ "image_type_ids": [],
+ "labels": [],
+ "cur_position": 0,
+ "pic_cnt": 0,
+ "video_cnt": 0,
+ }
+
+ IMAGE_PLACEHOLDER = "<|image@placeholder|>"
+ VIDEO_PLACEHOLDER = "<|video@placeholder|>"
+ IMAGE_PLACEHOLDER_LEN = len(IMAGE_PLACEHOLDER)
+ VIDEO_PLACEHOLDER_LEN = len(VIDEO_PLACEHOLDER)
+ st, image_idx, video_idx = 0, 0, 0
+ while st < len(text):
+ image_pos = text.find(IMAGE_PLACEHOLDER, st)
+ image_pos = len(text) if image_pos == -1 else image_pos
+ video_pos = text.find(VIDEO_PLACEHOLDER, st)
+ video_pos = len(text) if video_pos == -1 else video_pos
+ ed = min(image_pos, video_pos)
+
+ self._add_text(text[st:ed], outputs)
+ if ed == len(text):
+ break
+
+ if ed == image_pos:
+ self._add_image(images[image_idx], outputs)
+ image_idx += 1
+ st = ed + IMAGE_PLACEHOLDER_LEN
+ else:
+ item = videos[video_idx]
+ if isinstance(item, dict):
+ frames = self._load_and_process_video(item["video"], item)
+ else:
+ frames = self._load_and_process_video(item, {})
+
+ self._add_video(frames, outputs)
+ video_idx += 1
+ st = ed + VIDEO_PLACEHOLDER_LEN
+
+ return outputs
+
+ def request2ids(
+ self, request: Dict[str, Any], tgts: List[str] = None
+ ) -> Dict[str, Union[np.ndarray, List[np.ndarray], None]]:
+ """
+ Convert chat messages into model inputs.
+ Returns a dict with input_ids, token_type_ids, position_ids, images, grid_thw, image_type_ids, labels.
+ """
+
+ outputs = {
+ "input_ids": [],
+ "token_type_ids": [],
+ "position_ids": [],
+ "images": [],
+ "grid_thw": [],
+ "image_type_ids": [],
+ "labels": [],
+ "cur_position": 0,
+ "pic_cnt": 0,
+ "video_cnt": 0,
+ }
+
+ messages = parse_chat_messages(request.get("messages"))
+ image_message_list = []
+ for msg in messages:
+ role = msg.get("role")
+ assert role in self.role_prefixes, f"Unsupported role: {role}"
+ content_items = msg.get("content")
+ if not isinstance(content_items, list):
+ content_items = [content_items]
+ for item in content_items:
+ if isinstance(item, dict) and item.get("type") in [
+ "image",
+ "video",
+ ]:
+ image_message_list.append(item)
+
+ prompt_token_ids = self.apply_chat_template(request)
+ if len(prompt_token_ids) == 0:
+ raise ValueError("Invalid input: prompt_token_ids must be a non-empty sequence of token IDs")
+ image_start_index = 0
+ image_message_index = 0
+ for i in range(len(prompt_token_ids)):
+ if prompt_token_ids[i] in [
+ self.image_start_id,
+ self.video_start_id,
+ ]:
+ self._add_text(prompt_token_ids[image_start_index : i + 1], outputs)
+ image_start_index = i + 1
+ image_message = image_message_list[image_message_index]
+ if image_message["type"] == "image":
+ img = image_message.get("image")
+ if img is None:
+ continue
+ outputs["pic_cnt"] += 1
+ self._add_image(img, outputs)
+ elif image_message["type"] == "video":
+ video_bytes = image_message.get("video")
+ if video_bytes is None:
+ continue
+ frames = self._load_and_process_video(video_bytes, image_message)
+ outputs["video_cnt"] += 1
+ self._add_video(frames, outputs)
+ image_message_index += 1
+ self._add_text(prompt_token_ids[image_start_index:], outputs)
+
+ if self.is_training:
+ assert tgts, "training must give tgt !"
+ self._extract_labels(outputs, tgts)
+ return outputs
+
+ def _add_special_token(self, token: Union[str, int], outputs: Dict) -> None:
+ token_id = token if isinstance(token, int) else self.tokenizer.convert_tokens_to_ids(token)
+ outputs["input_ids"].append(token_id)
+ outputs["token_type_ids"].append(self.token_type_mapping[token])
+ pos = outputs["cur_position"]
+ outputs["position_ids"].append([pos] * 3)
+ outputs["cur_position"] += 1
+
+ def _add_text(self, tokens, outputs: Dict) -> None:
+ if isinstance(tokens, str):
+ tokens = self.tokenizer.encode(tokens, add_special_tokens=False)["input_ids"]
+ outputs["input_ids"].extend(tokens)
+ outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * len(tokens))
+
+ start = outputs["cur_position"]
+ for i in range(len(tokens)):
+ outputs["position_ids"].append([start + i] * 3)
+ outputs["cur_position"] += len(tokens)
+
+ def _add_image(self, img, outputs: Dict) -> None:
+ patches_h, patches_w = self.image_preprocessor.get_smarted_resize(
+ img.height,
+ img.width,
+ min_pixels=self.image_min_pixels,
+ max_pixels=self.image_max_pixels,
+ )[1]
+ num_tokens = (patches_h * patches_w) // (self.spatial_conv_size**2)
+
+ outputs["input_ids"].extend([self.image_patch_id] * num_tokens)
+ outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens)
+
+ pos_ids = self._compute_3d_positions(1, patches_h, patches_w, outputs["cur_position"])
+ outputs["position_ids"].extend(pos_ids)
+ outputs["cur_position"] = np.max(pos_ids) + 1
+
+ # Preprocess pixels
+ ret = self.image_preprocessor.preprocess(
+ images=[img.convert("RGB")],
+ do_normalize=False,
+ do_rescale=False,
+ predetermined_grid_thw=np.array([[patches_h, patches_w]]),
+ do_convert_rgb=True,
+ input_data_format=ChannelDimension.LAST,
+ )
+ outputs["images"].append(ret["pixel_values"])
+ outputs["grid_thw"].append(ret["image_grid_thw"])
+ outputs["image_type_ids"].append(0)
+
+ def _add_video(self, frames, outputs: Dict) -> None:
+ patches_h, patches_w = self.image_preprocessor.get_smarted_resize(
+ frames[0].height,
+ frames[0].width,
+ min_pixels=self.video_min_pixels,
+ max_pixels=self.video_max_pixels,
+ )[1]
+ num_frames = len(frames)
+ num_tokens = (num_frames * patches_h * patches_w) // (self.spatial_conv_size**2 * self.temporal_conv_size)
+
+ pixel_stack = np.stack([np.array(f.convert("RGB")) for f in frames], axis=0)
+ ret = self.image_preprocessor.preprocess(
+ images=None,
+ videos=pixel_stack,
+ do_normalize=False,
+ do_rescale=False,
+ predetermined_grid_thw=np.array([[patches_h, patches_w]] * num_frames),
+ do_convert_rgb=True,
+ input_data_format=ChannelDimension.LAST,
+ )
+ outputs["images"].append(ret["pixel_values_videos"])
+ outputs["grid_thw"].append(ret["video_grid_thw"])
+ outputs["image_type_ids"].extend([1] * num_frames)
+
+ outputs["input_ids"].extend([self.image_patch_id] * num_tokens)
+ outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens)
+
+ pos_ids = self._compute_3d_positions(num_frames, patches_h, patches_w, outputs["cur_position"])
+ outputs["position_ids"].extend(pos_ids)
+ outputs["cur_position"] = np.max(pos_ids) + 1
+
+ def _extract_labels(self, outputs: Dict, tgts: List[str]) -> None:
+ input_ids = copy.deepcopy(outputs["input_ids"])
+ labels = [self.tokenizer.ignored_index] * len(input_ids)
+
+ tgt_count = input_ids.count(self.sep_token_id)
+ assert tgt_count == len(tgts), f"len(tgts) != len(src) {len(tgts)} vs {tgt_count}"
+
+ tgt_index = 0
+ for i, token_id in enumerate(input_ids):
+ if token_id == self.sep_token_id:
+ labels_token = self.tokenizer.tokenize(tgts[tgt_index])
+ labels_token_id = self.tokenizer.convert_tokens_to_ids(labels_token)
+ labels[i - len(labels_token_id) : i] = labels_token_id
+ labels[i] = self.eos_token_id #
+ tgt_index += 1
+
+ outputs["labels"] = labels
+
+ def _load_and_process_video(self, url: str, item: Dict) -> List[Image.Image]:
+ reader, meta, path = read_video_decord(url, save_to_disk=False)
+
+ video_frame_args = dict()
+ video_frame_args["fps"] = item.get("fps", self.fps)
+ video_frame_args["min_frames"] = item.get("min_frames", self.min_frames)
+ video_frame_args["max_frames"] = item.get("max_frames", self.max_frames)
+ video_frame_args["target_frames"] = item.get("target_frames", self.target_frames)
+ video_frame_args["frames_sample"] = item.get("frames_sample", self.frames_sample)
+
+ video_frame_args = self._set_video_frame_args(video_frame_args, meta)
+
+ frames_data, _, timestamps = read_frames_decord(
+ path,
+ reader,
+ meta,
+ target_frames=video_frame_args["target_frames"],
+ target_fps=video_frame_args["fps"],
+ frames_sample=video_frame_args["frames_sample"],
+ save_to_disk=False,
+ )
+
+ frames: List[Image.Image] = []
+ for img_array, ts in zip(frames_data, timestamps):
+ frames.append(render_frame_timestamp(img_array, ts))
+ # Ensure even number of frames for temporal conv
+ if len(frames) % 2 != 0:
+ frames.append(copy.deepcopy(frames[-1]))
+ return frames
+
+ def _set_video_frame_args(self, video_frame_args, video_meta):
+ """
+ 根据已知参数和优先级，设定最终的抽帧参数
+ """
+ # 优先级：video_target_frames > (video_min_frames, video_max_frames) > video_fps
+ if video_frame_args["target_frames"] > 0:
+ if video_frame_args["fps"] >= 0:
+ raise ValueError("fps must be negative if target_frames is given")
+ if (
+ video_frame_args["min_frames"] > 0
+ and video_frame_args["target_frames"] < video_frame_args["min_frames"]
+ ):
+ raise ValueError("target_frames must be larger than min_frames")
+ if (
+ video_frame_args["max_frames"] > 0
+ and video_frame_args["target_frames"] > video_frame_args["max_frames"]
+ ):
+ raise ValueError("target_frames must be smaller than max_frames")
+ else:
+ if video_frame_args["fps"] < 0:
+ raise ValueError("Must provide either positive target_fps or positive target_frames.")
+ # 先计算在video_fps下抽到的帧数
+ frames_to_extract = int(video_meta["duration"] * video_frame_args["fps"])
+ # 判断是否在目标区间内，如果不是，则取target_frames为上界或下界
+ if (
+ video_frame_args["min_frames"] > 0
+ and video_frame_args["max_frames"] > 0
+ and video_frame_args["min_frames"] > video_frame_args["max_frames"]
+ ):
+ raise ValueError("min_frames must be smaller than max_frames")
+ if video_frame_args["min_frames"] > 0 and frames_to_extract < video_frame_args["min_frames"]:
+ video_frame_args["target_frames"] = video_frame_args["min_frames"]
+ video_frame_args["fps"] = -1
+ if video_frame_args["max_frames"] > 0 and frames_to_extract > video_frame_args["max_frames"]:
+ video_frame_args["target_frames"] = video_frame_args["max_frames"]
+ video_frame_args["fps"] = -1
+
+ return video_frame_args
+
+ def _compute_3d_positions(self, t: int, h: int, w: int, start_idx: int) -> List[List[int]]:
+ # Downsample time if needed
+ t_eff = t // self.temporal_conv_size if t != 1 else 1
+ gh, gw = h // self.spatial_conv_size, w // self.spatial_conv_size
+ time_idx = np.repeat(np.arange(t_eff), gh * gw)
+ h_idx = np.tile(np.repeat(np.arange(gh), gw), t_eff)
+ w_idx = np.tile(np.arange(gw), t_eff * gh)
+
+ coords = list(zip(time_idx, h_idx, w_idx))
+ return [[start_idx + ti, start_idx + hi, start_idx + wi] for ti, hi, wi in coords]
+
+ def _load_tokenizer(self):
+ """
+ load tokenizer
+
+ Returns:
+ tokenizer (AutoTokenizer)
+ """
+ vocab_file_names = [
+ "tokenizer.model",
+ "spm.model",
+ "ernie_token_100k.model",
+ ]
+ for i in range(len(vocab_file_names)):
+ if os.path.exists(os.path.join(self.model_name_or_path, vocab_file_names[i])):
+ ErnieBotTokenizer.resource_files_names["vocab_file"] = vocab_file_names[i]
+ break
+ self.tokenizer = ErnieBotTokenizer.from_pretrained(self.model_name_or_path)
+
+ def apply_chat_template(self, request):
+ """
+ Convert multi-turn messages into ID sequences.
+
+ Args:
+ messages: Either a request dict containing 'messages' field,
+ or a list of message dicts directly
+
+ Returns:
+ List of token IDs as strings (converted from token objects)
+ """
+ if self.tokenizer.chat_template is None:
+ raise ValueError("This model does not support chat_template.")
+
+ prompt_token_str = (
+ self.tokenizer.apply_chat_template(
+ request,
+ tokenize=False,
+ add_generation_prompt=request.get("add_generation_prompt", True),
+ )
+ .replace("<|image@placeholder|>", "")
+ .replace("<|video@placeholder|>", "")
+ )
+ tokens = self.tokenizer.tokenize(prompt_token_str)
+ token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
+ data_processor_logger.info(
+ f"req_id:{request.get('request_id', ''), } tokens: {tokens}, token_ids: {token_ids}"
+ )
+ return token_ids
diff --git a/fastdeploy/input2/mm_processor/process_video.py b/fastdeploy/input2/mm_processor/process_video.py
new file mode 100644
index 0000000000..91120096c7
--- /dev/null
+++ b/fastdeploy/input2/mm_processor/process_video.py
@@ -0,0 +1,205 @@
+"""
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+import io
+import os
+import random
+
+import numpy as np
+from PIL import Image
+
+from fastdeploy.utils import data_processor_logger
+
+from .utils.io_utils import EXTRACTED_FRAME_DIR, get_filename
+from .utils.video_utils import VideoReaderWrapper
+
+
+def read_video_decord(video_path, save_to_disk):
+ """get reader and meta by decord"""
+ # video_path = get_downloadable(video_path, save_to_disk=save_to_disk)
+ if isinstance(video_path, VideoReaderWrapper):
+ video_reader = video_path
+ else:
+ if isinstance(video_path, bytes):
+ video_path = io.BytesIO(video_path)
+ video_reader = VideoReaderWrapper(video_path, num_threads=1)
+ vlen = len(video_reader)
+ fps = video_reader.get_avg_fps()
+ duration = vlen / float(fps)
+
+ video_meta = {"fps": fps, "duration": duration, "num_of_frame": vlen}
+
+ return video_reader, video_meta, video_path
+
+
+def get_frame_indices(
+ vlen,
+ target_frames=-1,
+ target_fps=-1,
+ frames_sample="middle",
+ fix_start=None,
+ input_fps=-1,
+):
+ """
+ 取出对应的frame index
+ """
+ assert frames_sample in ["rand", "middle", "leading"]
+ if target_frames > 0:
+ assert target_fps <= 0, "target_fps must be negative if target_frames is given."
+ if target_frames > vlen:
+ acc_samples = vlen
+ data_processor_logger.info(
+ f"target_frames={target_frames} is larger than video length {vlen}, "
+ f"will sample {acc_samples} frames."
+ )
+ else:
+ acc_samples = target_frames
+ data_processor_logger.debug(f"sampling at target_frames={target_frames}, frames_sample={frames_sample}")
+
+ # split the video into `acc_samples` intervals, and sample from each interval.
+ intervals = np.linspace(start=0, stop=vlen, num=acc_samples + 1).astype(int)
+ ranges = []
+ for idx, interv in enumerate(intervals[:-1]):
+ ranges.append((interv, intervals[idx + 1] - 1))
+ if frames_sample == "rand":
+ try:
+ frame_indices = [random.choice(range(x[0], x[1])) for x in ranges]
+ except Exception:
+ frame_indices = np.random.permutation(vlen)[:acc_samples]
+ frame_indices.sort()
+ frame_indices = list(frame_indices)
+ elif fix_start is not None:
+ frame_indices = [x[0] + fix_start for x in ranges]
+ elif frames_sample == "leading":
+ frame_indices = [x[0] for x in ranges]
+ elif frames_sample == "middle":
+ frame_indices = [(x[0] + x[1]) // 2 for x in ranges]
+ else:
+ raise NotImplementedError
+
+ elif target_fps > 0:
+ assert target_frames <= 0, "target_frames must be negative if target_fps is given."
+ assert input_fps > 0, "input_fps must be provided if target_fps is given."
+ data_processor_logger.info(f"sampling at fps={target_fps}, frames_sample={frames_sample}")
+ duration = float(vlen) / input_fps
+ delta = 1 / target_fps # gap between frames, this is also the clip length each frame represents
+ if frames_sample == "middle":
+ frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta)
+ elif frames_sample == "leading":
+ frame_seconds = np.arange(0, duration, delta)
+ if frames_sample == "rand":
+ frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta)
+ rand_offset = np.random.rand(*(frame_seconds.shape)) - 0.5
+ frame_seconds += rand_offset * delta
+ frame_indices = np.around(frame_seconds * input_fps).astype(int)
+ frame_indices = [e for e in frame_indices if e < vlen]
+
+ else:
+ raise ValueError("Must provide either positive target_fps or positive target_frames.")
+
+ return frame_indices
+
+
+def read_frames_decord(
+ video_path,
+ video_reader,
+ video_meta,
+ target_frames=-1,
+ target_fps=-1,
+ frames_sample="middle",
+ fix_start=None,
+ save_to_disk=False,
+ cache_dir=EXTRACTED_FRAME_DIR,
+ frame_indices=None,
+ tol=10,
+):
+ """get frames by decord"""
+
+ if frame_indices is None:
+ frame_indices = get_frame_indices(
+ video_meta["num_of_frame"],
+ target_frames=target_frames,
+ target_fps=target_fps,
+ frames_sample=frames_sample,
+ fix_start=fix_start,
+ input_fps=video_meta["fps"],
+ )
+
+ frames = []
+ for frame_indice_index in range(0, len(frame_indices)):
+ frame_indice = frame_indices[frame_indice_index]
+ try:
+ frames.append(video_reader[frame_indice].asnumpy()) # (T, H, W, C)
+ except Exception as e:
+ data_processor_logger.debug(f"encounter error when get frame: {frame_indice}, error: {e}")
+ previous_counter = 1
+ later_counter = 1
+ previous_after_flag = True
+ if frame_indice == 0 or frame_indice == len(video_reader) - 1:
+ cur_tol = tol * 2
+ else:
+ cur_tol = tol
+ while previous_counter < cur_tol or later_counter < cur_tol:
+ if previous_after_flag:
+ if frame_indice - previous_counter < 0:
+ previous_counter += 1
+ previous_after_flag = not previous_after_flag
+ continue
+ try:
+ frames.append(video_reader[frame_indice - previous_counter].asnumpy())
+ data_processor_logger.info(
+ f"replace {frame_indice}-th frame with {frame_indice-previous_counter}-th frame"
+ )
+ frame_indices[frame_indice_index] = frame_indice - previous_counter
+ break
+ except Exception as e:
+ previous_counter += 1
+ data_processor_logger.info(f"error: {e}")
+ else:
+ if frame_indice + later_counter >= len(video_reader):
+ later_counter += 1
+ previous_after_flag = not previous_after_flag
+ continue
+ try:
+ frames.append(video_reader[frame_indice + later_counter].asnumpy())
+ data_processor_logger.info(
+ f"replace {frame_indice}-th frame with {frame_indice+later_counter}-th frame"
+ )
+ frame_indices[frame_indice_index] = frame_indice + later_counter
+ break
+ except Exception:
+ later_counter += 1
+ previous_after_flag = not previous_after_flag
+
+ frames = np.stack(frames, axis=0)
+ assert len(frames) == len(frame_indices), f"len(frames): {len(frames)} != len(frame_indices): {len(frame_indices)}"
+
+ ret = []
+
+ url_sha1 = get_filename()
+ for idx, frame in enumerate(frames):
+ tmp = Image.fromarray(frame, "RGB")
+ if save_to_disk:
+ save_path = os.path.join(cache_dir, f"{url_sha1}", f"{idx}.png")
+ if not os.path.exists(os.path.dirname(save_path)):
+ os.makedirs(os.path.dirname(save_path))
+ tmp.save(save_path)
+ tmp = save_path
+ ret.append(tmp)
+
+ time_stamps = [frame_idx * video_meta["duration"] / video_meta["num_of_frame"] for frame_idx in frame_indices]
+
+ return ret, frame_indices, time_stamps
diff --git a/fastdeploy/input/mm_processor/tokenizer/__init__.py b/fastdeploy/input2/mm_processor/tokenizer/__init__.py
similarity index 100%
rename from fastdeploy/input/mm_processor/tokenizer/__init__.py
rename to fastdeploy/input2/mm_processor/tokenizer/__init__.py
diff --git a/fastdeploy/input/mm_processor/tokenizer/tokenizer_vl.py b/fastdeploy/input2/mm_processor/tokenizer/tokenizer_vl.py
similarity index 100%
rename from fastdeploy/input/mm_processor/tokenizer/tokenizer_vl.py
rename to fastdeploy/input2/mm_processor/tokenizer/tokenizer_vl.py
diff --git a/fastdeploy/input2/mm_processor/utils/Roboto-Regular.ttf b/fastdeploy/input2/mm_processor/utils/Roboto-Regular.ttf
new file mode 100644
index 0000000000000000000000000000000000000000..7e3bb2f8ce7ae5b69e9f32c1481a06f16ebcfe71
GIT binary patch
literal 146004
zcmb@v2VfL8(?2{ad+uON$H1LAY>MfWC3h031oSW{sOP-O*@UYa+3X
zh(|PQ->#F#SC?)Obx$XXINhvM=Z4Rg-K;X~U-`jO6UuosEl$l6d1-Zm@3U3hJ
z!)#QOh+V*akGR3H!_EZ9eof?goX9dgZp+e`D`8HtMx;G_fhYUa6b@s6(D@gNDY%Zd=Yu5EUFoLlX&~^;&DO`-s;9~S1orU{8E=C{YV)Q8~+5dvRg!?rv
zMwh@ZM%QpLx`~U?ZTbc7U3v!h1ujM=NU1^Mad?6@pus6^3hL#o!i)ObJ$+
zg#xe4s=%$rs==+z>cDNl8pCbIn!|0y+Q99=I=~G_|75H?12m%v@l!2v-6Toq>oxGLTgpThlI
zdxi1Bb$B(?I;_U0)}b{_>yYX_tb?n6Y8_PFv<|FZ
z);gei0c-y%rnO%s)7rPvr`A4|O>6IprZv8TX^pFpWbIXGogS~~^zuyzbKtsR1s
ztnEWgYrBxi*0#k>Ya4s0wY5FT+RAQPTiUx=Tj0KVG1J=2W?GxtMp&B!nbyWZBdm?A
zrnO0;Y(^|iPFo<0~F@GXQ5e_O2E{=iUNW^xX!0uL*0QoQT9~MXLRtZX`CrT
zLkmEvokNWpY1KH#v@b0f>s8kv50wCYz>nU6Zcm2JJ4NT|bGilIy<|CAE*8v6unMdO
z3uB#FPu7>c!zQtXYzf=U9P9ym#!Mc_tMf>HoWIXMdnR9+<_p*&G{`Lq1p
z;6|>ZRJ2~v$wfC6eN^-bI6yXUo4+l$Ex*lX3kC-&*=o8tFv_;ow!@YL4ji!^e}x0x
zTpY-ijRW!E00RdYIPe@CfFPK^n>W~F^Idk^JOkOd1h@$J67V_TQ^3c7bAb2Fe5QwK
zy!`X!UG9bZmM~L$ziiCvz4-3Mw=X_?ap1+S7du~Ud9nUQ0$5v+v=qMZMj3xSfBF0w
z(etB#RKQNa=I5K>t_Cay%tfk3fH2@ah;F{PdE(}fo2zf~n=5awx;En4$ZJDk(+#>d
z@LK!i>@kxGZeemX1;h=Y_S4#B29Dh`V@Xr!a!
zm^dy@h?8_yoQC~z4wlGy^!yL#L;46d-6!H*Wz`8&d_Z4`59w?4-;c#7uu(41Mfyg3
zE=hbr-@>B*4w~(2`d(as&G!TSC@zZ6=(6}md@H^a-@}smLHr0?@QS!Bexf_D|1#)T
zafR+d$375O=^;I$$Mi&8qu<2O;yV2
z%tP6Wa-9@Xviz(7E655-Lt3P#^pI;=8CI5+W93-|Wg|g%RZ^BCY$tD471m>QRs&jh
zlk}0ka*13j{bUZgS#FVQSZmfsu9B-^U-`>ia+zGty0ES+LKc>RvWP4ytxa^VJrSn9ZN3&JrE(s_DNm~qvGdy=nl!G(KKh_Tm&
z`(UoL#)bQ4!~Iw>InEWHgR04fF1%nCZBP)tvYIm-_PVz$%x7I&ruhihYE`gE<7)l
z6k}X?J}N3ATzGyeCfqa^Ae+eN3NJ{-`6CxzhzfIeS%DP7UHr$~qbMwNCp{q)0KMeI
zOOP-0loKyW-h6-yFGcxyxC<{$#V`+WrU|72ypRholht0VEEU2$!P<%oYx-5ON%rGBXA
zztt^^8~^y!>a*{ZlQnsQ9N8*=sczTpmJgAs-{q
zHnHIP-zDzvA+~H%_@>w_8r*e$Q>yq3hPfJsTNi}
z)ogSaVuw&4txHC^o^@-Ka=PvNk~9C2WTU38bT;Y)tPk=UtNUVa#Pvt6w%2lqN62X8
zue8(%;EEoVwiC6W*6`c8%Fmv&s#lz*Y9!n;NYzC@Jrdzcrs-?0&VZ%D?2s||aG{u1f$xP
zYq{44Ubnmpde`xu?S0HAk58men$I`B(l^w1u@A&)sxA1?@Va-9yy6VNR)=e(A@F?o~n`R41CZ*V>{f0_K-@;@zLDPS*fyuhagsbFxy=z>QJepV<)
zp|XXV6&hA(R-to+zAW@}p~rzTFg$Q`;Hkm|3U?^Hw@B+Edy3pBTBGRvqQ6)RS<6{t
ztaGfptT${GZR>)9gIWcx4oWZ9qS&TlSL~(i?d+@VY4&gIKNY9qfyG-DA5%QF_!q^W
z2m1#X4=x)V5j;A0VeppV%fUB;pNE8oObba2`7-2QiDD&2mq;y{tK`IzmrMDS8eQs#
z(q&5bDSf8&?a*4GiJ{laR4g;G%x7g@maSHHR@wKo|RTsx==Z|a@)$&EAOs+sY?DTU8>Bj@T5Q>XWOdR=-oDV2uVf`qr3MGk?wcHAmK5RrAAI9<`d*npo?@+9hiDt^IBt|2hNf
z9H{f8Zo#@k>YlA9>P6Q(RPSN^()Hh}|8ZEkumufdg9#1(XgH$bca2InifOc}(Tm2>
zO{mG%ranzaH9g$)&t@f?HElM$+016^npbH)qxt6MCz}7&qE3s=E#7N!v!$(Nx0XX%
zPH8#6<))TDwF+$2rPZQVTU#A!b-Q(e)?Hg~Y~$HxXq%~RmbZDo&C|AiZEbDi+wN%l
zW4k)-*0j6czI6M#?c2AX)&6w*uiM{h|5pd!4&6HJ=z
zxL^4A@Ee`OI&bWJw@a-qGrN4zHCNY;U5|D>-}S4mPyhLf;1S*tfe~dQnnuJ#OpI6_
zu`l9S#QPDCBLgDqL`Fo$M~;eI5Y;fMPt?q)^-(*b_C=*dor(G=>g%Ws1?H1Cl
zeYc6-9Nms|d$-%i-7a*y-0fDk``!MGX3^f!xuXk52S>M#9vOWu`g!;8?lIldy1(qv
zzQ>v#cY2oU*}Lban4&TLVm^%Ju`Od`Vu#1hie2A}_bS}0U9Xs4bK?reb&5-kyAq!(
zzES+x`04SR;*Z9kkAKm-X7A;_|LoJL&(l6n`#$Jbt>3_Y3;TW2zi9u3{df2OYe3xr
zy$8HMkPR$2u;;+J1CI~9J*d>6HiO0wT0ZFE;EIE14o(~Va!Ao3ZHM$9vUGJMg4
z`fGIe(X&S%9sOubnK9kREF5!U%+s-z#?~7#QGCwPyFhw3UAGR>-)FsyglyiPu_m=PQ`cLdT04N&n7jU)OFILNv9_F
zn4C5x$CL(BMo!r}<(sLwrgoZ|I`!wNFQ%287C&vyG{>}i(<@Jpo<3{(h3S{4-43-fEw
zpFMxk{O1co7K~i5W5Kh9r5A#|3x_W}x=1W)w5Z>rZHo>s`e@PRMGqFs#RV3ZTijrA
z_~O2cCoEpDc=O@|i{D@T!{U2O*pfU;N-U|hr0tT}C8L(iTC#S@-X&+2Tv&2z$)8Jo
zm)e$ATiRl2^wObArz~B*bm!6|OFvn9b?KvJ#yZq&fTq}yNsJ5cTis%(XS4>&4Y{kwMM^}8Z;>wCgD~*)}Rt{eI
z!^(TBXjSf2!K-SlYPqWWs$r|9uUfe(an*@cU#z;m>i5-Ns|&9#zq-NdPOJN@9#psvHh%5cwR6{QT${S~?AmYF-dX!%UCwpI)>U2CVqN!j!`4k*w_;u5x)bZZSa*Hh
zAM3r>7hYd}eS`Jk>-(;sxPIaKt?LhO@Yv96!{`mOH>};TXTzBd7dG79@N}c!M%%_p
z8=Gv5*f?FH*_&9==|HaFWGwRzCy$(xsM-o80)i`SOITgq)|uqAv;pDp9IBy3r~<^EQ-
zHTTw#tu?o{*&4HTs-rII@+qG?vw|i_axIK7#_3bUTM{ggtefsvVx8K_S=MJA8MR!!((P&4P9RqfZ
z*^#hg!;X|4=XPA$@$-&5J09(Lx|8qp-Wjm7(9U8zOYf|-v)0arJ6rDTv2(=E89P_)
zOxk&3=a)Ob-+684FFPOZ{A-urF59lkyPEEb+%box1zn?oW4LOe~OSODvgKC$U{(*TkNQ
zBNHbkE=b&zxIgiH;`fOeNtBc)sYFt(q=rc?le#4JPa2*yJ!xf9a?+Wk3rV+=o+kSx
z=S?n>Trs&xa);#Z$-|PTC9g={ot%<7D8h2-1G_xJEUdH0msQ+H33J#F^H>>0Ud
z)}DoXw(dE+=i@zB_B`5a?9IP7bZ`B=o%Z(KJ9O{Zy>s_&+`E78`Muxoy}Q@k7qBm6
zU#)#@_r>lTy>HIG#rwAJJF@SSeOLB9c32#R9AzC19N~_Bj){&{jsuPl96vf9q{x&4
zDP>Z^Qo>V)rz}X>l5!~J!;~LW?x(WUys0Hq>!h|zjY}Pqnvl9a^=RrhslTK?-|xS_
z`2HIETkr3=f5iTo`&aMZv;Xw|3;S>He|8|pfzSiZ4n!Rod|>i{r3ZE#IC9{V13w*j
zbkK6J(7|#C8ypNj*!SRsgYyq=I=KJf`GemdynE0*l>1PLL$wdJJrsLr)S;P&wjMfk
z=)*%l9=d;+AI^KY9HKg
ziXE$VtmUyD$3`5Rd2G$GJ;zQTyKpT1*z@E5$BQ4YalG~MnB$|4&pN*5_}=619{=X}
zo#QW01e^#tQS(Hr6FpCiJTdFU+7tUuym#W;6W34NKk@9O$I09$ttUfIRz6w#WTTU<
zPIfxk?PT1^fhR|voOp8D$+;(&o?Lr!-^q7RUObt8^1;bJPx+q8bE?Ftnx`6^YH_N=
zsmN1KHcW@>eHJ~?>T+?^jD`ZoxXMYuQR@93Y-Z&
zQ}0a2Gd<1>KQsNz$}@>)4xM@T%r{{zSFc*RV%E$V)2B_HGI`QFZ@)Eh!uWAx$BZ5|
za>VdqLx&6=G;l!wetrA&j*sgV8`HB#_vmg>kr7?HbPn&-p?$lyZCbZ#*`j%~rX{WZ
z-d<%`ijQ|Ad!ydoWy?^Cw+~RCvSpaV(#YYV10C%`ZFq$k5!9+=?
zLk2fd5F4Km=Z=U(7I+HJB8yg??5#RPMcA4o#OS9G68_4aGroq(QOCJ32X7P};bH~&(0SYZwdfY^XuHip?GqB>Q%D3Oo3OwXrV*pj+(^_U((dRLY7eqU^hQ=G
zSY8bZk7)$V*Nrh-Gn8OU^QT^bxUTjz=E9;P9JZJ~kkXSZrWIkUhv28JT9DDXdc4gOC@ehADP7yF*GCo82iY
z0);wkv%@1&Ipd9D@M5+&;v&*)&O}GT@SZ3xjXT5qb-I)gRS;r0Vx%)BOvfNhBFxhnHrbiFq${;2V*FLi5p1|1
zunsgk|G*_QWSF-R<{9P{=8GK|XbTmTijboW+m{&a_ho)8Fa>#{1_(}LlT*CH0zoZu
zat0m6>n1Oz!X{^ifpV2BTSh3-S@F(UHRv1_vEP@Fzjl$Lq53OZrb!BK8*0xI>kbiU
zz$PiIZD6MoE_~!rnX39+{FSPr|S!$i|)$
zYYB-7O^9$PAycAd_pb#(6~Qgy>`h}-3Un0ww?I%_7h8;@S4=2Sn}5>;MBCzGRcgwY
z!bNZjlfkSW)dQz2z7B7D?}iQ^d&A6_`c&T;V^J|4_J$6Y&&lT|_9nIh{S)Hsy&!X8
z?IZdI_KA!|-i|PPtV7xx2Bt`Ai2hW7p%P6}sBI`}(+X18F0_3&bW2rzTS7u(TS}M=
ziH(a@_l<+l(-T}#_Qs7>H+4VJ#FpR)i;asxiYAddaoI8;A)$#q))o(Sf$D-+o$PA!
zL6j;bJSrl=H{Kp^2T#Mo5@J#9KwDg7U_xY^<}t_tX;ik1k)@AZ>WHge9~{>Q4mv`w
z7<(^gfa;I0hxL6upbt`I53{#a#evwCifZ@?E$vO>kwQUiyhA|lf^6~F5ut>H+G|bm
zkBN{)W+EHfRA-*xUt39vx;&p_g*v5+IF^?&xhs
zaIS=a1Y6An=&9~du`(pCOAH1A>_BL0(@h-)nin-tv&d{GH98aupG9wiu^}~wW=$hIc6Y7cl
z{{B>g=YLliL_xQv!%GvQv{8tsBJ7`w!X^mSKrK}P@VK)YEmc(vFdBV2At5dnHdXf=
zs)zc9Qoa|tLds6c=HzO-CmUr8c<<<8y=ym?*k#w!sCMofAH~)Q?z{$>5qz^
zfkJc9oo1Sffv(g})`7psPs_P0T6YYUnR;$)Xe6+v3Sy9|sRCC|@^ST2-`BLhD|=^~
zdc7KH&&*5>^!Cgg)m;knh50K34U`yS^9Q?X>81-oO>m!3D}{N4xDr4j{ZY+Y2?;)S
z=a6Do>i-B)7-@5oBB7mM4RE}TrcX%l^m}uR=j-YH^aHB%qm}(KeU*!=J9sy8_%u@X
z3mVYjrSwoawC~&Jl(<1Ev^kM&IMhL?maDTm2gw48_h2+00mp}GWu_9k`PN5wX=iq?
zhlWSIjY1Ts&Vd>JSfxJ)=?PL0s92|E&VimWG&Is_a=oo&W2){_Q$uaG{;;zeF<4s|
zD1qCc4UpP1L@T)j7(D%BF*HLDDZLX}0Jc;|We>yVxBJ_$`?-#DB4l?;TqleHGB~16
zV9iL(B+|@Z3M+jD)^H5+fbayH%|92S5^MpOS~zB?t|(n`b{&RM+!EqSrD`xEG$Fy6
zPFdl;JfT%5FhR}HylVt{gQms9TzuR9H=w
zvdW#3S?*Yrt0+it7o0|Y-&m{?Io3_N`d?QyjW8P9orK5`SJ?^C2~n6J2BlF^RlJKH
zb)TcKQsbJSrHUXTF%NIoTz^dMu>){_+KfGKbL6iyR!*WJ#*frrKBYda5B1|GXeR%V
zN{A6uSSCwSYS^n%82iWCP;R6j2}lC;0Q3Y@1*`$g0JH@R2gI`nG=rs3DVd9!
z%XnHP%hN3E>uTjuo(3B=sEB+{yN#+e&IqI3avoq1?nlxD;~?$kW$CaSKvfL`;q7R*
zMWsc!F@?q&zS4Mmgoj)RkLlsc1o^;Co}A
zPMBCnl8>jFGKvPvcW9ohK;7ju)S(`A=O0jAgvH8!G@m`91?-M_Qs&V3IuA*6s%7U;HI#m2grLlJ{AN)vukiys#a=_1H45w+p=SsY@
zl{={$>eF9Vp`0v(mdSMLCL*b~r3FQcWuW;C^_D|ulBFfWlV~CK?N5`;>kEpT71oNS6LXnDp@+j?;
z@6tTWyVOpm&^vMoHA7y}avd#~A5c$YE%paJqjho=tp)GxyfoR3xwHjomUwKY{vM4f
z7UgX=W>F7*)Vyv?qCRpdMH`F2&&Sw{D9r=12E~c9lqeR^0!6plG(m0wG%_z)no}zaQK-?ICVDQRN}k)Xr*0b6GY*i$
z@S&|1NtHbsQ6S{8y`sxVqzZ;7wXu|?aN`o~GbYeVk6BoDI!z8s0OhjWMH+*)BEMxG
zCFn3@H%bnsUf|;f5e>dypmLTV$|wCP0$*ILmz@BcsEl!+Hb{I~q{77aRMPki^*f3C
zDC%b!PwOokQJ+7ksHGaxMF84Tyk$20c<`g2c^mhAE$G{p8+c}p`9kI6;YI5#WoU}*
zYX0C+hSH3`(3iial90EY9=`OE#g|T4zM`eZ&$QCZpu-+tLNQ)926;
z2WW|BMezFxEdk$#gQr8l=UTEfmG&r0RgCUrHICCtxq|u`!{863>G0=RDo}lS(!8wm
zj-s{3OR6C6gZKMUCgcU}w_9eQ45gRq(FWBIy^}+th;zkhUx;L$~aFqR+
z%7`EuCiCEnMZBwb`y93p#~yFB;~Dyvcm?2F07}d$|qI{aNK4xQT+I<70^Lwda32fjy*fKzk8O6A-7KnFw&|u>awC34LklLRQOS+UYq4
zZF?7ZDC7g<5Bir?+9?$JD#&O~-Hmo=*L3K?�egE!tu*v~K}gWeK5h%eT<6pQ9}X
zpua7m0meeK63uq{FLRa~t&-p_h{Go@P
zuo`|G>e5_>QVto8ewr8Y{jj(7Z5l00KwsIYG2|yf9tZv1F_s*s79QM)pJ#-B6e@*z)tPCI#2a1@)mQbp8OooJ;0vcz7%WZ2aS(u
z494OT@-2$>Xh;o)Cc~{|Ir+36f+64KU2K}59e6I(+(_5B9zu8XH*%IjSZKxlMb_ttZ@K9#G@18n@Xq^DjMqqaFs)IM^t)waro)a=VneL0@;18zDan
z&}Qp^!}dXZtQ>~EJ)XMCZ>S$+iCJoq7rq5_%UA^^XWDjzEok(hUfQy;-owD(=gR>GX9wnjnNm#WT+o`3O`{j1vI
zAAQ)s=(F>|uS2NETKJ#BPlcZW|4Z;QKYL_8g`WufG|wsA#bH~GXXj&dpT}g>Gf9t
zx*p{&1tbDK1T6+&mcUq`$Ap74O^pw3(DW%9U#C-zEpAX_Ms`qqaYObxWWz0Lyi#>i
zpvEy5=zjCSoM$O$uHlQ#O~!KbsZq%sWArhf^Ht_r9gjJl?jL9;C3}$J!%Ai?ug(3G
zjHq@sFwawZL$|TA`<1N^`%K&Tu;caoMfGhpPr}?x={J;hL)rMc&%@U?zUq^hUnm(>
zW$HZD+!N!2Dqq?0ne?f2NDul_U!hCc{JscRdI0n(+O&RBeO}q`iVoL&(=|6Nt@MFV
zwm$j?`r!hlm-L(yw!TW^mgU!dw`?;tS66d*8|Ly#_rs=BGs&q#{)YKb`^<}DI)4YwZCJjEPJ@xuf28Z{Tu?eUr(boU{*UemTP=6(|3
zel9a_HD6OSxP3JjRl3AIhj9Ct6RCNN;=AI*-+auaz}tRm4yETgn7_O0`)1$#@Xg_G
z(C4q!&;6Xce*f&7_hdaY?VsWm@6?zKe;e9*=Ii=1+db@W^WALsP9C{=<(_N*oBz7r
zTm(Iexw(6e{!hMJw*KaCh7N~Z>p7>3|EkX@`E}2i)VO3A<^=_a6C&Dt0si^OlIGWL
zc~bpCV0}dQ)j9LyhHo$KI|TYjQ3xT(|W&{xzVth$9S-LTAnwr$i8N0
z`NI4|$H9J6`d;f$t>0nO!@gIxex}}5a{#52UGoJ!_5j!8gtGIYztvnog}cWBgxNFm
zS9Yu_12(+QSIrsJc=b7amA4v;RQ{NYXxm%Gt1y(IZE|J%E1O@<8`3bxg3YL(N4aV&
zaDz_o>RXw81mm768~p-SBqmMXELmVG+qh}#a5gM%{D?tpSlzv4M@wwrSVeMoZc@=whRoj_vTlcDeA5tq_YZnoYdCQqy5h0((O&
zIC};gEpUQKPV%Q*6hOHt59LLg{Pk~LvCbJ#}>hW*2
z$e!NgU(!qDt!j?dh0HSka|wZQl}5&~wsCQThq0#dgZlSnVevzT4raCbjEIe66%Y{1
zN^7^McJmDyGI}uc$GJy?n74MNa_OaZ|3u^ndNdRsWeij9t&xM{hSAlLBP&#-OK>aE
z1-O;zGq_dg1GrV`UAWcgINa)VXyoW#Bgrvx^steXI7;QPP1nWAg(4a+s@(wX=F+ae
zc5`Z1ood6tRpsKFcDPC&nTsd6HC|Y|7VUay*U+w~c9o=Y@*=FqK-QG+#PRJaLWGtv
zNEq@A)UFR)buNnHzADR_xA2vTg0>XqqSq7tOtv8Nj^j&6Z?>E!5Wdr4E7(f5imhgA*xGC$gnq{G=c=uytP
z6vQ)%cZ&1qdu$oQm-6h2`tBb&)JL7&c{8%GDY#?ki|#OvuUYcQWVwgWmAm9_oYIiQ
zC(FIypgPweANZFCeYFTm%!`#94k`+%zKeU%z~gy;;(Rb4OrCrwA4gs|dt}n?uG$(jbR^$fGy%n1ru!2djKRKX-CUQ|tX|E__rrDo2CPv7oen7B!kS(1Sei{j=gX
zD4UG-oXh88+}#DLa5j&tmGI>>PVJh_=WulflfXA>t@!k8u|@HvaZeu0V|Xtf$EWbA
zd>Wt5XYiSP7E<|A2`WjURGF$$Q+ykR^>BRW7EYbBU0QW
z6Enj&dV$3#X4sA5Mz9fLlt7GyU*zBL+dN$i5F;~FA{=>xGr}&4XO(WLMAa;0{BC3z
zmyBJ;3S*uTWrQK8Jn|0YY@3`a`^he{kn|My#95IlCW|p*kch&m4MlO1gExPGvd-dk
ziGKKUs}3*5{n=0K6PC(mvoUNC&ZUZAZCP=8N?+j2t+()97S19wCWxEZihH8tyJR5$+yXm9MA7hE7W6zr#(Gz2GMCGjMldEnB5T
z-SGC$jWx@}nrC87HKyeO>!m0|G|(a0d9er~iC8T|?&3GNyW~8$d-&&YcSADrz^3pN
z_jSzADn>liG0t3n0-h+Zs&KS;8JuiqL%+o{@)W|7MFkx{8Ez8)8163EmMZs8;O+)M
zQt|dfS1-8B!&?p&7NWw0O{c}V4d@mAI4$7>?vtgG?>z#&5-AGl6soON-`M>c
zCC5~Hyz^C*_JfB>lNdFuChY^-6yTDC^rK;#x
z9M~m0!rjA9!gX`n(H;VaYrVZlbIMHwo*=D&Jag_h7v%
zE8puXM1<*(?6R{EA&GK6+$8Y??k>DBP|qHKn^}9DN}*z|sTlD<$2fU-8F-?+qQY@*
zR27VjC8?ORmGRaOw2AUMg&F~pxC8DktS+hiHJ9X%ke0;Ex+_h&N0icacejYD{~i&n
zTf_={H#Cu3ijLzx8LPo6MW9ZhMt0q+@GebZuhVo?J%M+;it4^_lXw!`U9vaaJvb%=r#j4{)5=$3R8QI?xQdZV
zu3wjU)uzc@_2xvpk5`oLfV)e^!QI1m!gbbD$^ISyDB$ST$W3wi_1u&sBx7!-a_a;)
ziEn|s3$t96+g7;lTD+%1@TSQvf2xl6O5A3
zChnMi6i;=KpR$mZUFN>f!*m)of~tH3=tfN_nV3Emkm1e(3fsVKi%at7*dJiv6rNwO
zl5khtB`)Kk4^%o;mP*@}KS%ss9qKBd{YZEDJl=-4=B;r4VM7eCCVR8U98=UKpo0
z7UfoM!}*QHaC&2L9?Wy#Ogw*{iwE%ByfZJsOY%~@w2>R9I+n#3*X4OdUYRcztwkHr
zR_Wm?ox+8DgfGC1#5`
zI7x0UPEwnXv(y%fMMe>^80X3@!?|k9#R{xzCNF->|;$2mMKZ!H#-H&*=rd#JP14WyW!`s6I0ar$tp@RdDiC4OWZQVGUVh
z)|9nmZCDgviBq+*o#%zG>yPsj{3JicPxCYUUH%^Iw{uzNdu5&N#lPTR@e8o;zT=np
zkNhWomH*6dV0M_!@9+$M54PVU{)GR||HPc}InMsVK~};L9ylA&2j_m}#F<}taMD)+
zQAiYqwP?favbYF=!diP&g~NK
z;@r@);+!}y-WNZKE8?oSCVm#z#SL*&+!D9lr)P-_@vFEm9*W1}H|Z^XapqNFX_uvN
z9^q=aMsAQ>U?<}=Q{AIIp%b6dQ+!c0kKJW=*e@)d-DbDgO?HD_XFva+`ZVVPrx&14
z^To~{t$H)JID0Yru(S7O?~QpfdofO4{fFNBrXE`XeYO&MZB<^4*WlGfOVR58+<*Uv
z-Yeeuf9}8Q-_V0~pH^05+4?XqsC%)Y^q}tPO8@yn1ERk}
z3+8&Ye`ogYtiJseW0p6@s_g4=fv`l1U~CJ~bI(#3t3oldl*I^D9wSploK{g8V_j8@
zb=9c`&ibf@F|Upu^VG>6Q!t`Ur+KuH7UNtyoK=Ujkq_Y1<3n_q(g2u}i
zTl$W^Cw-C}T_&6xM_1_@{Y=;C2Hm7vbeq!Y7o6=@8E3gw$ElRHaV}*e)`Ydd`M<3(
zql#oH>@YjdPT_>z6*x6-P1f0dd=uZyxA3ie8{f`%@SS`Y-<|!ezpQiq_?P@^%oM-n
z-}4`Es^=AcjbGm~Z-{rsJ1l5Q9G5?MK!T;jV_zV6LC;M?Bg++J@Z=CCw184f>
z#z}tpML`iLiU_L+!pt%l=lPYwd46R@c~MbR7F9)cQB%|wbwzyyG5e&Tt9WHpE}bIv&{YKG(T}jpX2udP8M~Y%ZC$2KNnx%j7^-zCobR=
zzE@7*!|8kX!~^k2Jds|~2WQ?DlO<(2oOZWfZpIwBxbF8@_l2I`&S?%8qc~;b6~fcWRY4D~^*9Hw-7Y*BjV-#1N{c-3r<*sa+eAs&x~U
ztMZ*%NadjJun?_7j4t^qJP-a6Cp#-!6gmpp-WzTWUDjEOfsb`Sb2-wJIUCSLVXywF
zaQ&y&$SmZ8Ia&bDz)oe;*$g(5&BEM#4ohGQ*h2BU_ycFxaip-|Vo2))*B_U%^KyZ%
z09+i>oJvxi`G@s2BHT2WE+SGBg18>fS(0<)X%fhWoa#HuEn4wwO$
z1wg$)4+*TsqkKZ?2%y}i4JCn>`CBVqdPS?8a~<+|i|m-)=s$gmx)XKY|EfImqBF=`
zW8O4>apDNQ;|jfF{$~D~?MbyB`L{pwhC9f-?#8nDYHb+5JfIx&H{~iH*Aw$Tu!n%p
z%-QA^9exaV*mtM@e9g1wZroos-*<)l2hV(Hp7;;Ze~U78?UDO^HwOIHE6LRAz5X*F
zC@(9WGxW`_`Ph8)=9vFJ6m=Q*_tZ`*UJZA*jJv(uvF>p9{XhBU-PfOdHKpk~?e*2r
z|3B{B<(l`v*^Jk7%;MqSM%@(94Ws&}L*7FpryOb*Mk2
z_#r}mMYzM&cOVd^z&JP8Uqhqdg`?Z*R5NxygByu
zP}P%FYrxHtyZ}YN^AB|V%b$4*J^gaz{pr_x>ML=N
z%?sv1^DFe08|GKe81q2(mNl&^)chg)Q<>=w{yp?xL(qc%%b)q7J9YM!Hq+gq*?sin
zF;3diqxGM89&gDr|6U$PO|p$QW(^gk`UP~un9O7<>~8kE{|!fb=rZ8n&C18!`?B(K
z^IP@5fBO4>)5(9YhjTo2_v?SRKirb=y6u3R{|DP4tAAu+|5W2QhI6cfC_7SFX3CCK
z>x3M0s47^WtAVvaf2^l8!<*F>xC&u4r8QOvJ79&dFjfe=z|M=p6$Il6oV|G*?Uz$`D6=^GGNtJ0EW=d6QJ7!DOX$NLZHEAbi
zO|@wkW=>6LH|F=9u=}Zp=tak|+BSgBV3lnsPBot_rsGue*&=~H!dlr<`V=#udpObj
z0oL_D$4uxEeIcH}7XA{p_)EGdYhv~J5`L8+jtSXYj=)c@Opp^;fP7oN&G4n2oW%0r
z_Zk+ly!fqxLo86H$uwq@N98dVBu~f_%r4K!_gHaxPM%{WSHW+rN$kyp!)W+0eYSj
zz<$7Q^5n#N@UK(=>%#Z&CER^>p8~N`{1Ce!l$007%JE~In*AI54a=8G;ti}Zn*@1>
z#9OiM+zPmocppf74E7}8eL7q<56cT#--K!VX1*CANgTTdF#q3=y$T2LMkohX1&?9n
z`#AaoM}Ihn-3#Z@D?G3Q_!(B*Kj&ZI9sQU5OXTns{|YI;=6IWe6~PO@FY=4Pzv15i
z|CWD?oWE0Vx3HS*=ddxPJAdyC(K
zo6ghW-a&8VScABW^cg$@_^B}GZ#s&D7keLJ`A+qre$_QdLD1+wTK
z?um8IO2Ab=_r!{472vA3dt&Xg8gSL;J+TT}1Gwt>o>&j91zh$2Tr8^`ymJ)EylrBdc?pA{9=4fs7dK|(>fmnNAkEfJ=$))v6
zF0Eg3LBBi!Rlng~wXdG{`{{YVpLi~wqZM9=7g!H?3GE}HeQJ`BweUVu%G!9tBxD`D
z=ajOptP8vzc2`PSU)Bd6Cc}W^_)p*sWkcYNWFz2>Wn`WmDkIWHaE+@wK0n
zEo2MeEwQ6q%2u)!@Yb?5@HVmy@V40HEAhKD?SQwJ*nuHCsyDVWT)o4^9)~V)BV+{J
zDD3y;GFnE%?ScKjBhNN93qDRAByj{7=8_A81Uh8IPej21n`mAWx(VpISToY#_j_q$Ee*3axC^E
zFgZ@`U%;>Ej0X-)jr0>Gc7MpX)LsUqwegz-@8Au-Md@zfljUUKQzhP*$?0-B+?jGF
z+}UzA+yt2bcdnd^-2n6CJnXxeFXsbaAQu2%C>H`(d)0mQnoVB0U+%~L$^#Oo>&t`k
zAUJkd;@vmKgQK9~m^=o0PGMhxMV`f1PF$W>Z@T3d@(VommHZ0s*Yazm{7!yH{_=bI
zJ-$G~sDN*}f0RFh+RO4X6_h{8pHSizymu}ruga^yui@QvE{qx1u`}xi_DST2X3cRjfWdi9pU4ygiLLya8%
zdgSo<+sIK=j~ss4Mh<~DKus||HNzEvF{B07-<0LzsYek{J&IWLD3VuOF}d}~;iE^6
zVtVB8ag7|xqRFc*8k-&=g7nznt49T2t?LVE{q6;e<|#D(GuSmnTv}e)HC|fFduzKU
zhqh(>wSMPXzvsZJ?{?_o9eBI#)bbu$%lm09FQDa*<98lTK-U{u*L!MR@6_@>TFY~-
z<%QPrTx)ruwLI5aUT7_!Lu>f}t>trQEgzt@d=9PU1GJXUp|yMftg36EM`?KjE6dk`
zD_w73jrk^UrST1{Hs1!W^uB?0=U;#;?Qg&qdyHB<;ZHDD{RX>?X}gSRyNqeOjA^?J
zKgt2U@1gCo0@^OisqM0y+Ahnf?XsM(%dmG#j}4w!x6T1vjS!w#!S)BPewoHoj}R6;
zHdyqiV9|D5Zf(cq)^=QOt?T`@uJ_lv-e2o_53S`rw0`%{n%zU|bPuh~J+kR>Kdr}o
zv>x~Suk<+H>p+LL7OgS1DQ)hp^|+VT-QGgkfL^cxCu7(56j*}!vuav{wb1&SYke)WzUEqA3$3rY*4IMoYp(UR(E6HdeJ!-U=2~A1*#SEooLbwEov`yk
zYTa$f&e#hfwFWn2SJ@S~(&L82kIMm9+T4)cWH;bSryH_6_L@tr*$w>mTujV25YuQ@EdTI^JwT2b)kURu>V0R-{X;_Q0yP+}9;Ct6V{DRuM*oXSQd>^qN
z$Pb`HKa?L*0cC^3g$=I0@xbvTlT8=e}`3?4x>*YzvCBUkGzu&QMh+=cU7CTjZfGg}j{E9!nDS_31g`)z1_jFBxdLblT*WG9S@5f~G@VJz&4(GTx%
z)VPOHPrZeDgq<2wN7<2<;
z&n>kx7GupF{v_M`Oy_Qefb73c=GwL3e4DA>VAjighxw+r?O8ixobS_zX6|uu?rvEM
zeXQS2LjUTwZjg)@(6F_kP3uE*HiDLH294MX+OHinT}NoRaA>wJ&}b3RWKqyy(a>By
zps`|LiS~g7+7FiJ09c%ZU}+A4g*gnCq~O*X)OOIizOr&?E0*-u?kJ
z#>dbUUqCB-4V~~Uw7_N9U{_#!U4z9X*nRUi_Q1R+)?!tO0mGg&xQE$qSOHb{4{`M9
zujWmp!+s|7F24W$Mf5dqily+^nh(V~;Onsps8Zc?<@OM%9wF5|q3}Z21Xi@WnOA5Hxck8T6gH&=>yJKwD6FS;
zHy^X<<|8%(FbgmnFvonvKQ|xqFU-fH9^f~??`VfV08h49HUkf1I_|aXY3>ak08(f;6@&5fcZr;te7_kv;edMbOywNQ_JA5Hq-I8+aDbB
z$J=gyyzTbK+iri9(ovgq)CTr8;0_=I
za1ZbR@W@OTzhhnF55QB{1s%k>h27c+nQn
z4$vOZ0nic93D6JFA20v_85NLG0T~4qmqEp4P;nVlTm}^vLB$PFaYJ51+3Gu%9H`w!
z@TZ%(4O%4}9D4}vKLmds>$W(9yIZ(Rr=956`OvTPQ80F&lzaIn=F`~qYyJf3y$k8R
z3+es;*n97QD6ahvbk5A|E_JC3f&v1{Djk+ymkt6-u?q+&U;)9kw}`z)jlH+Dt0@{g
zdSgk9i5fLAxkil|6HB}j6H_#ava|1V&h8=_bFcUJ{(KKTJF~Mhr+m-1f4}EUGqATA
z^xX^$ZU!80bNw(+z#y&^WjU^`L0N}uy9~X-A6&p6T)-b(z#m+Anc*m(kFwU#n_q{r
z9%Tc{MwCq`n^E=}u3^>fHGV(J0hA_`gD8hk{t2vkALTI05tO4SCr~~{`2^)8$|;o7
z*a_rQlrt!2QOi|O^h}47UOV?$1wrNL>!ZFOvX`*V+xL`
zIHuv4j$;OnnK)+Qn2lo&j@@v~#W4@Zd>p&u*aOD`9E%Li;0n#)3eDgO&EN{n;0n#)
z3eDgO&EVC|;ML9G)y?45&EVC|;ML9G)y?1-&EOc#;26!|+s)t_&EiXjyJ92Cbd;GW
zvr*=v%tu)WZn6kv3CbR{k2wdIX$HS<7LVZf<2YhoVZ}H@Zry?0x8y*`-hyNC%HxJ4wkWiGh;TQd2ZehdhsMSk31q8k8Rsid|Ul&eSh=aj^?eHqseXWX`2bq_PY5T
z9O$V(6V%5)b4HRdp3VF9;Jy=r6JvrYe+)QmV#b0
zXbd`hrkn3GBpJ^PM^PX0V9XI~%<4tdhYW+Ze=L@cD7!IkjyOC0L#!P!c;qF}@1|e;
zke{*v87%JstLzOsaDAEK0CHN8(Sq|Gn5*~b8T~`N{2rqyj?O*4-vTBW$3efdYxH{C
zzyE{>ZSOOlKMR$=-tjlB+<4BNt(kSt`|ZW9{TKS6kx6PDuWZfxeIHr=A3w2kVcjAN
z{HeUx|CaT72)Xaakn_%LMe!C^JKRNv^snL{;3J48;opDG)^tMt@or$y9{iJI{lOt{
zocHn120c55zS4JKi`;4alf6)0Cyh2bU=(uB7WwQ7c%zM(o?C=1h*iCg-NY2g=6wSm
z#BBoBD`7qP+t4F-3A=E8udo-__X+zDFFPO{z1%r!%`xyUJ%w>6gM8=xxQD{X2pt83ixq>51%rqcaN!oBEOh)bHhxPsav5;rSNQb)0EW<&6tTkjbPsc5{204E@2UN)QLgeU4cc^lk+6Y-I(Kk2#`F?(G@Nqvh^(=1IJErXUQx;
zkLG%B16J&4lihs)7{IWcj)*%2nLP_SwFP$o*?5N`y*~h-ISy;?l;Jw}Dfs3YaJRFN
z?zT)jH*4hwp!pw0{?QS`OWcQs8t#N)4)?L43QujsQ^WAoC_GgLE#NTd_&D^`k5Lbg
zRzAW#bj{lVA0
zIqdhqAxE1-OWy<2=SjGTOh7TCbqiA6P$5pm(3(+DVjC&{IxBdjM9T
zh0AE+GFrHd7A~WO%jnyEwDJ>Lxq?=HLMuO^mCI=5GFrI|uM@?zc4*@Y+W6Kqr?=p6^!X6;_z~JUfiZr9drqR9g64DDa20Ky#r%GW`TZL0ToDezoAN#^vm=HJXzyFJ
zcNLJofHslwfT!+(&r-~zGx;&bfLX-#xeM@~cS;J^)H+GJK9-D3yHz8rh5A_G9G7(Nk#3z=LP+vEl0U
zN4Uw-6#P-h`laIIBB$4#ee6qu$vOqz%v6jCL=FQvCojn`m@J8TSr^cRy}Os}fx@AQvm&l>I<
zvrr{iGx)|V92^06^Pk}p!#n14dPHXB`&5TE{z-K>IvVETIx@lW`8vuWdWYc^b_c^K
zc9qT}FpFLN+Yf3X{q2wR`5R2H1pfT>?@31KUw@-l3=_az{)1*~xb!#A-D6MRZvC~@
z`ZQkwzrr6lz;pfpQ&Z+YZJVw}fp^S=&w#xn)~!G?(6uSgvjz?08Ld2H@fe$VetS=M
z0UQZ4hQe$dhU2C=Hs1%^7Pc<_J=F$g@VPKHKlP%==VhNi|5?~h8z>i|s{jnU4St3u
zaQb%0`ag~D%#t#^ggYivXhB}W-eM~OuwKAiAH|IS%OCQ@skLXM056IKPjbTH&BPz&
zLFSxhXT18jcA3Cz|#wc-qvdt(h3Y<{{DW3$Poe5$iX
z?0sHFyU`AO>gH#@{J(kDSQWGo{5?&36nFm5f42YUC;#R_M)xLp^S{&4-&ESVGVRIl
zG_(EhN8Tzg|HADJcb??)8*4L31UpJ~)Tdy&$~+#_g8F@lKU6;6?d7t!dA>ayN8E@z3C1eg
zP^_bkXkAI0hL!O>NpGy8Eg_|(>?s@F&A2myKv$fe7CsfuJhR@^xZ3ox@U?ISYfb4&
z^rzRM-!nya4I&X`q8w{bZLkVeDcXtlqJ!utI*HDrN^}w3u%A)~(L?kUy+l7zEe41|
zSf?5yhKgZgxTq1AiYvucSh2nyD_1vR?W!P?VRbWIlero2kV7}3S#Tod9I%#X{_G4*bbh)E0Rgln=+QXACzk#r%7S5Yj)1=mS1gDH
zV#&4yS>D8zxZ=DWX~#KXR@|^>v^#N!-=RHekLw=91D~G66E%1dFPwW5Z+!X?A6y9_
z$O6X~Vlyys#=d78%kB$SNl!i6r4Og&M`@DMJnAqylwT
zlWIIQj0{67!^v=5sVDWg(m)ze4_zB;%`*B#mhEl9^7=%U?QOxby)Bqm!411{C}FeN
z@yJoejvUzi3j27taCX?2Lyb>=y#K@oJ97l!*FZjy?t_b6Ti51bGL)*V7VCOJk=hyhWB3wbX*j3>wd=b}#Yn%$$ik*G1cfxge7`_$0
zMb`3n!gn~oA>6?An*wt4u%E&$_-k$p$i~LL3g6?`JHj1gmmz-z=l6trCSQ!mGRH(r
zICe-8@ef%>GEv640_)}QODIGIu3O+e1_G$S8lN^IRy2beY>}s=6qUHgUc{!$(tjGt2sBrEgy5QVhbjMSm6?}T)Z547*j2G_r#V)0IE2xOw96&*Ss9!Cr
z;r-|$cENd95$jGwe-S&kfwls0Js3OQD?wo)Xg5?0#d(+*hVyVS9OoKQgYyV60$Hb#
zVkCU|(PA`ej)Av>2Q@B*M`^jZ9KWs*SK!xG;wt>QMcjh1(ACY>pidd+4jQ$=9^!T~
z%q(^xu;*;B6M+MEj&_zgbfQjyW54LBaP-YFg0at8df!E_+P3I1n=PfB}sjOy$e)wwXLdF3I@Rd
z44`-{XBh0lFqmgp>%_2D#jw_uVJ*+FR$y2wGOV>@SnJ5J){bGVBlx=;>ZG_UXSge8
zxGON+6&b!dFnqOV*l7>$9|Z3V#ZfDUqddb=D~6*y!%-`SqefhGhICkod_IbW){qiw
zvDf%|aXoyW8^jH`z6ozzkTL9Zh2#)81;b7^hMn#VFWZB^Ia3Rpo26Q=ny#9x8mk(j
zN>$l7ZFHRLkZM2LzQ`^{*-hEbcBOrheUWV*{j@K#skFXnUu1pII>G9M)oYfIEZ18u
zwfN2AM~mYY-4rnjANfsre*p2C!bdh&RwlC%FUYQ;XA_a1RB3Ts;UiqMT&iqm@gw!3
z3R~@q=sB%dSZ=x=E?i{a@vCi~xE?DkjNg|d7t4a5sBFjg<-3a$6+Wo%N4~q|Qn3=>
zQ9FJO6f4a&itG8lwD8^OKi1-Mnx(1mL9LbM!cV0A;eD(R0@JpF(-LkLU}D21oAcBl
zMXA*R!
zzUd3_O<#a-`U3pW7vP7!06+8vL>y=g{729X^|{eri=Y*E!*3)4Jxxa`#P>2B8{oIS
ziDwxk#c;mKpBowhieU3b@mM+7&lOELr!9MU}%?ApR|F5wGlg@c%UFE
zg}J`gtj&dWe&<*lzE7;(wL4CxnOpP>p0@*j}n0MK>QYr
z^9aK|;Tmw^8)Tb(3v9Z9<4weTZsT|l$A|Fs|BmtqF&=`fGuYAKLOa2Qc0&8M7M(Fp
z7Zi7V?||ZgqBdLrN7@OFbOD;OH8f;vXvWsyPCI4jkIcq!PloiZb=eT?`%MtmP5y^oPJV+76Uc{6%_AH8lyukWMB_tC@q=-qwv
zj`rw2dUPK>x{n?;qeu7Aqh|EtK6=oM9yFtMx;vj7bZZTIoQX0EWe&<*lzGT6hC~9D
zJph%_EN04w)X<f>g|k4!Cv?dK{8qfl(*X&mTEgjLVqM`w+Bjq~ZtQ36vL6E=kAxm;7GP<^#|s9ru&G
zUcd)*`T%tL0Cf5Qbou~!`#*r=P~4_m=RVGVM)?(m?m6@T##4heA2n_c)>-B9CI`
zV7@r-j1rBq3hi2O=P}C{fKjn18Ne3g-HEoyPhN`S7Vt|C-0>avC3J@iI9`Hp>N3ue
z*~ndkSN|T4zu-v`MFUwG3#v@RF$2c}91C&16vt|mdK6@-37;W{{R*OMG&|BB#Si+HTrb73
z8l{%CdJC=IHMM#Jt=_`;Xv??I@~vlDzJW2{1lHe1wCW+UU@2zA8O}m7eh;bm4W!}c
zkcK~?=FNcj=QO4a53Dr`auqq`8a?%-8`C7tk}N#^H#Gpys0MfxbafPTbrf`U6n$+5
zjvWPU9R+P26=jA`(9dQ-s2LDy79;T1&*f+<4mPbFAaxr$v=WfIjUDftVB0F;eNw`@
zwMLI_p-0&54p@E<-lku0|3f_aCnDW+Y$EhDYQegJLu)uCpm`hP{T`kjB_MhmJ--F&
zyMx}}0tMay3~s`&qXaj%gYQNOc-#g&Zo_M%1WaxNCf~zfqXd_?gWYKjkBJ@T`Zhcy
zN{r$*Ab%Sk2_-xccJM&h!P}sOMP@C+LWY;X8Y8;}s=WgoxQ!9s0_EPpjNC@gFChbg
z9-E=J-9v=wCrEZZW`savQ$U~VfLRKLYz+nG#o)UZr5m)md}y!(4evvD`Ubj_1J{vj
z56V9eUiBp?^#YEU0h_BhM^-*y^)q1c8!+oPz~X0M)^C8r&%mq)z^dOc-d`}@Uw~Z?
zFy3E)(+@CPzhaC(1G9d^7=ObU?*qGj19ts}S^E{U_A6$MQq<2F%Y8`eOBe?-1Thm|
z0oq@opI-p77tmL#aehxKA)f{_$$Wun(D9gOd-VA__XXzdOPpW8In}DJ;{03CBs5yg
zuM?BRe`4fR`dVPVm}d}2H(YCvqX&u?iZ@CuuBG6Zfn%28Tg>xy%=5SC&2{wTTgu)jF*8!hzxJ-u86q_$&rmlfgev464ET*#MN6gss
zWy@zwwwUD##cs1qaRW@LT+yJ%5tvOXUyPW3wpGTY;kR`BX2g1m@pt|rZ%T1b8Sbe>
zeS;y7j2I6O3v%ljmP0O><^KZ~on-KOKCHg^
zTUc3QT<(x49l^QHvN+ao7w?*2`OZPg8)68$!9r^
zN)+?hLvS93qCqi^`!JK&g{|E6JmSq4P%eXh+HzVWMx6W`j5znVqokbKh!OXJ5hslp
zAyMhy79*&2aFS%;dRq?u?5Im2d&_X&GqT}tIr|mq0eHpzRtViPoNR?qTdq$v8Or%>
z4etPJ?gML10&A!xa3b8_yu)R16nc!?nILH7c2DFLh2rz^8Gy&&HC{)IzD8bOEL-2bj
zj$t^45cY;bCKX!bWzkTU@bi-Tpw!E)kY
zIdQO@@B`}Rpc`o&+SB^_jc}Hp-
zd>y<|oN!KvFL6LvN3hh7Z6C!Sk8;(I<>%{n#PNmfL$)(2>QRAgCo8j~qGTnH7Rp8&
z3oOwspyNL%ouH(opn#+#)HhFb;yZynWazQYZtM<^*G6d0grg#r;Oh|S%|3~X
zXFXZxSuf}2O6y3;EA(^Fe0XTbftJ2#Gh$z@12#dnRVq0vOH1qmrQ;lF&m3YyG_-^}
zXhOp(FnDhi;vI<%og%#vf^NS=x{0WChs8Lod<-=>_c3fw6ATK3Sqt>@=L4!E456Z<
zd_9A2A{V9YAS&YGECTF(Mdy&nNO_2@j__Vy&hGAlvkc9i(S$f?9AjhIiKA;QJ;%7{
zJfM&WKRuJT{&|FDUF&u
z^
z;YXPv`v`W2FV-()ak(5In-iWZi%;(w?yBr4lj*{3DP+Tshua?y*Erx`XjrjvFfJ}C
zWAFX^qUmU8?;|5ITsl1R0{1u(H5r;cyQFMZcGj%Y(mB}+e6ypXGJSlrq9U_=`M4)P
z&CX#zW#Ol2{B%YiJB0sO%*`*JTU0c+IKPk{Hs%KxX*5N_`60y`O>s*>_ab_=$avV?
zJs4MuLb`|0dxZSQi&8}$xG_i975^$b&ZTmlw9dYcoShGmyN8AbI5@;R=@jk)1|J>}
z9ugiN;^0bWH5y}#U?XG`2qH7jEx>`fRLGQ=QwoeEI;Xh6{)0mNC`8EG%gGrfJ>v?x
z1a=SS-@B$iojW#dz-uFhuj?7!FJAg4JJl&^YI)C1FXg`B*CWx=$};!s5wCoq>#;!S
zvM!)cUVKGVzsR8SA#p7y_}BU_o8L{hp{Un_;>`Mzz^@|9ySZeI&B|Z=@q|G;S0!1t
zvn(tec5r;j!ijX8Lqs1=V)SS6o&efAfoTNIIYmmUX++MAkd}~vQ*qyH$)8^-9}bRU
z@t)j?3jwKPx*Z*(qFt3dj|q1K+A93~cz$-}>VIUZ<0n)lB~?v`S7-fWbtRu^b(vHU
zs~zfzK^t#NpB+{nmd@RIHMBv!gIL0BMEz>iA8o4NNzUp|Q91LxT%k_l9f3YP$R*mv
z#nF+EHa0N6Iw^JNxJY9I1y)~68>QcOs2hT|z@>u*i01B=z>(@5(nDzz>hHi;3cY3T
z!N1_FwX!7&QB>NWxD
zA1BWJSJsoh($@Ml1EilzNk<7v9gnwuEJc36KE@U7_bTwT9Iu!=p|L+f*&*e)pms5$
z#hou`5^Udnmu!1{MeIb!>>*u6L>;*pfT_rFA!kCvs(>qW#F2=#mlO}}80dOs#emc4
zIe`NzqE^#q{P6x3CBGl7VZP7~$d0zbE2Vb){?)5-#}aVpv4~Gvb5^?rOEBjEaMN*Z
zSn`tin3#B4Dnn9ILV~qgTCbB~h5(L-dM$S$w8e!aqhesO;Go~Ki7%J!{PPQW5Jo!+
zbYTs;aK#=U;Qs+~FGrb9ZsY}&2Vl~a;A6o8rYs4J%To{>B7y~9AE3X3v!_7uU$oYa
z-!`o0qLBsuDdV>d?X|d}hyOv>z_dP5snr>sT!T~lM5b3~_y~E|c0XL1=3lzcMzudrcnPm*p`uB^-Sj;P1(h1^@1j7zcu(3NcVV*edtL-`Xr
z1nA2mJM)*Cqy@Z#?7*XJ@$B$9xo-aPp#iyrVx)|D(BV3
zW{=6to!w{BrxPl6jM2`R{!*gUkK)ZhtiFDP);nNjslV2xeFsl3Ac2R6qb4MztF6w~
z))J6KPr^eepP}LlG6dsc?DTov@%2?jGErij0m^1f%$MFx=FF^^njF*6yZgAbriRnA
zXPy|L8NPko#*GX14GqYc@{d6S-)zioN}JNF*QB((1!aOJWn@-QU0h;AUj4q>(*1Lz
zr_^S2sZSj-Bc*!dh?uCFRTX)<va_Tb+zI)@gzZ>Qp;)zFEI3u*;-^s?8%4
zVs-C~>APW4zST~f>@nR_t1~U$Q7d6BN
z_nw%SF}815IK`b==$VIX7xrmrueDVMw)61`6gW{AXrctFa8PQ{s2Jv}m=JI63MFw+
z0+sTNSU~O(j!~V2WPAH%Cq`?63#y{EgEKmHOdXV-KdYC#Qkg$$aY5b2>ae&`|EM3k
zb97wN8e%WowPE0<6V(?gR#ZoX^_!TJGGSnJ?u0DTu&FAfXx>}>Yxc~|tJr#?y3bnD
z<^Cet=kAyzKR{DXDVN|4DWXnbRKoP46cxzbJ5u!1zy2ZLzvTzn&LNKu$l8JHh}=kw
z)(*I84?komr{%o0_O^+Mor6>=F)~P|^APEIs1$sCNtn~-Pinga^%xwZtzuJ|mQ>eEBlb-hy}qP&V^vt3Zd?77o%InOg=>G`
z>$I-_>LXR(3|c!RN>eg9CuL$~Y*??__~I=yd-PoRPQM}h7IYu{kMs3uBme#7n-zHf
z$Sm;qTI>qs$2lTErQjS54~0><-W3w1C$6h;<(e=GGi`4}sWTh<0Z$aM_a)-LE?OHR
zD3B>?A;SyAAVNYIUnwRk64{w3BFV@6?v}GcxW0R%5YBIw?i?qRMtv#WRE{Rl7_g@;
z-VJgYIa<`pQ0|p090lHIlt2`1I*>vli3;guDk5~(#|ph$w(}=n#>aS~IwH*;bBVN9jh1H79^}G1Fb?z8
z5qmmf#t7-u$=_P%;9%>*)H2F29W+$<`X=$wPk6TiUyZDrsa|cZ6yEC~@y<&Q&JPV6
zm>D-jOPct)gQKSIF3Ztrqy0XJiHwSgiHeLNYs)89r-T$&q-P9{^6!&5a%=CBx5o5e
zH8VleC`~XMHRxfT6fZv~TZPPqA)H$4=qQtA6*=iD1`oC_E%oq7=o$*aO*uq}1|MMY
zj8_0{pzudKX#}|mx1yNf#AH#q2~Epr67bPcj!{vaDF1MjnQwJ9J)IFq5oV~U54O{$$7)61hw
z8(p3sL4Gf3T3Z@cGQEKJ*I&pf33E)YU)FQQqYq|h<}W#=U%K+~NAt4$JL!S~i#OdM
zK5xH9e13SVr02$)(se`Th&;7N(%{VOqRBl2l1FbDG<4U(N=!JYXPt
zf^$*qvE{mIokfMh%GMTvYa1KhLf|10EQ}n8%E{PRK-CCSuryeDYM8rJNjffhe#8e%
zC9g;x!c;!!o0hY(kVl_0J1=C~qXNGD#*Gxewt?l%MQd*0b!f}2s~@jc6F)ztjUBJ+
z(7wHrwM8e6wuZKhhMl+Cn_4=Q(RoAZ@MioSs3v$vKlvfr+hpAFJEg@u=}|m6BXLT(
zbn_Uw+OTtyw&c~i?owVI$tsza9yhsCy2q38U575sA9s=8wk~F9PC-@3qaS5Eg9a|G
z?*B@Ot-i|!?U;hXvD$SpmR0M^<{qk`e4g$nbP@ES2BlfKsq6(^Cr)Mw_UL$>mywvrd4NVyRX03F9WG@EVl)Uq+wCUJ}
z8TU5zoATkD%#8zg>o@kLw^+W{X~ZOo-MG;{VhuHf-1+bD|oFe$j>vybeeru=+z<>&NI|&+%5)*1$!E
zifnYi4nP~Dpmnk`?3m!IAI<-yH+dKIBR6j37o$ePO~kBS#jFKTSOxg|%ZxL}CN3QF
z2HDDLqjLv4i>VAu75&TXMLS>wSKE(0_v)-~UhS8^^76}Xemm9im~H-|hN|V|A@L*D
zl-4w6J4rkEf9adM49Y3pO~|VwUnSh`(ov_T^_bqH+x+8`M!j@mLAGz03&~theU*SG
z-lk@TCwhoo90h8g@j3=f=3_C^9L7pvNohbVZ~4LVm^kpLkVGM8_A6nu`+=zAS$#Hp
z;XWb^BTT>=f6O$ExR352_9^ajqB@Jcjg7)mN8LKWo)GFJ0s@&83A45<*c=^uZuacY
z#vV(b&^>QLCfOz`9{;)IL`}&*7nh8UNttFEjf}NU`AG+@yEy}f;iTqRfjso%L&Kj!uJM`hycN8xuX6&T)nx@Rrfzi18tx
zUTdkAcW@ANa>{Th6`&3n@>Fy^g|&`Ct14yIrDRq`YrmQ`?W+yFd#(R!+N`hE6(5PL
zURK)wm718Cp|A8WTUr$%$V*<6TK1Kf?<3-CC9e@-e|h6Y|6wfw#YxobONo3Z@g()cuM@s)V#=E_K
zbL`lgeEf#&#-iMrg_H&BPEK2Rq6RZH!eAwnWAp;#MF_PdJF2J}5+GOsRXOM)fB`kZ
ztMDF9@`8{MmJ{7{X*v1#QvkWRmI7pHS-)3?03b{IQGmQDeLbFd^IyL(SWI|%3UFx$
zx$3KRZ0~L<*w|QGa~<5;S?I6~fua|>O&N|+HS}f%FGQK*X&L7YicoDM-@SHYT;Zyr
zQBpildJmbOnfh|Q^vyA$SlW1!)W~)Y{&3OY*GF|wZOECN+w-N2M?dn*iGCdPQocXDdxjC~ES-5P&1b5UK*>=bqJwEmO#jET*jeP~qGhnof-jjNiL
zS+y{SG-!Hg<2*a|sGixY*OK80Q%>rNrsgCKOApS~M!EU+9zLg6za`btP)F)=hi7`n
z3;`Tr2#Tu!M=E1L-&k2g*`bk@nRGhJJirh64-fN8AFnfV3dZdhV&vQLtcuxU4svVo
zQARP)Trk=vY>jd_lNm4EJ-YPgmzV#v;7HF^gR55c1U0K)yIfay`8DBEi(0olJ9{~N
zmps*$L_O-J+7d9IlUoo=NIi?TbX~=G266Qz{8aMJ=lWYKq%CAyyoDNb1K0SmRj8$1k*FybkU_NJ-XBs_ZasVDc&vU-&_M
zskSk(#ZNfcGJ2e_V0=+a3B-Un=h2lVvW^T>mPw!R*78PbZS8?;5tdts$+@5cUsicL
z@YY9;NOR?ltN$FoT5$kl>Iz?8tg$b)N?l>9FKV!K9_hJE3u4Qmx9t%szb03e2{ivL}>S*I9+LU!IlYtz?+;C0Lu3PaNTWYVVV0>1mC=M;
zc+h^kup*{*eO1l6VaYZ&$-~xGS8p61bI>!jwnt`7c96Aw&ZGgk!_zv-xku;E^M0za
zCzlsx%|1M?Wc=YdSv{7XennNdxGp)guqOKF%xOi5wak-L3aH-$)YVkp!6)ae(22ka
z=4>|lD}mz>tkM6`(%_R)8?t3PW#f*Hnz(mdeEhh*6Q>@o!-bUkZoNjOd2~n{H8W~p
zV)^S;y;pu%=U?~X%HB2aHKf(1q>Y3rnKi#5jp7IOqyF3mVz%C*=wc0+DRhJZj0q9+
z2lVHQ@Dnw8ATv}ZgV}8IL`O1Xn-`)FZr8U<9J!&Qdd=`88|&oZYikB>7@2s`JF_;o
zWL&nhO;A=%W**$m=Vif&X`|NJ@CaRq7pBFpjKCCcCV({dF@-?RkJjI&Cvvy~Ga0lt}Vt-{yA==C=G
z6(Dq7(Jiv2sn_Na9aFrlRFUrSy#jCR_la+h*-~s_u}UU}^iS0-T7=oDka`KfVBAjd
z;U+M>f%#FeTy?k6lqhZl?h4GM=vmu
zJohqC?mFX3?o?tp@Ge#goxtgM1}>An3u)nG_VD7I@fw)3nN2U}6Yc0Nl?9ChLJx+Q
zXJ(8n3fc|OA5gYpxIUi$xK~X`T=fhp%PF}HW!j}3XTM)zE67bvouwc{
z;6Z
zS(b9Z%!&amJRrH%(6Z#!(MCHf>!8p>>LUGN_dY3zm0l_*Ek-~|agsv%@uTm^;KwI;
z@pU~`Wpgl4NpIGQ&ED3jFhU0qF+NB8umGCO1mL0wX%P+|wnJA(KG?^{)fzwrC7Pzt
zfqHcCVXQzS{BM7
z4LulJla*Me^%e%TY!Krl8{>qREUDIq@SpUmrW5Yr*YCB)+~9%BtM#$`$pKYe{d4e!
zAtTP2`t7E*b>#ojV`gQcH_z)W&~As%U-3>TZh7ZPH~xZmbf4--v`K9``~4jptaWNR
zlcY@eJYi2J0fa!mQQ&}c8M#$SdgH#4y8YAA)2Hp#>Gn-aZ}Q6@ou4-o~{SQnircd9eYxs*$l=PXCa?xSh&w*08aR2Fk8s(6v)S;`bvOC2gTJs{TEy{j-
zOMP6!rt0c-by^#1ZQX{N!JF#i(a*Zvl1bg1twXvE&CDB-C%pGO76yP8
z%z!K!QXHnOhpx)-iC@k7=>qv}E4Q?iL4db%7L$(3pxN2vNmeL~dJ>Rn5B5TE?dqO|7)MKCmBKb^DiZvV=>%w8rSCTb_kp!Y9OSxof&k49);2YC
zO&W?*w27o5spUs87UT9k_?~-y(BQ43d(0@)GYuuK54pXEPqVl@~(uv7m5PPq&($#iHg^BOAj6=hd_u*rPFopGE#W`WY
z4yr@>V@#*fkEg6aIjlhQpjdp|39X?ym?4B%-MfED+DPgyJbLsMsg*W?;f*Ep^>_7G
z$VzD#-wB&*09GziuCNpJI>93XdT@17!3Jz6*V$P>MPbmQ6;q3u83SsyQ9X&qkl@mJ
zp9Gzdy4!{24e#cXt|S(|isZhDE=!18=qyRsAe?&a)ups&q>Z?e2;TW=;`){YAEx6)
zbHFpC0SLR%u2DL6(AqMaQBG|}ITrRMQ$h4xgHhPsa*uzgj~K~+Jhn*hv3m7t!K)=_
zuCQwjOe$Du(o%ktJP~rfGsa`(VQFR6p@X9nfh8D@=m@+z%%g&E65Q-?%teON{H&-b
zCk4|y;aG{J4j-DX#E(~x>tDEI(a1q5>vpK;u1g6l&0VykP(47NsOx-W(z(H}mQQS~
zJ2!Oyfx^#zQ9Z6B5&*X1I;vii*GuIU}F$yRR(={+v1@4TGo;;@MP0Kt;v&#j5iOb>|&>z~YbpFKFb
zTS|CXR5@U=Kw3qr={pJGnXuVS6xd%XB;@K@nxUj|LeLz>D29s|-v(uyyh>@6sCwKi
z%MVOS3Jip@Cm@5*8rdb-{RDo45U#V#G-z~R|2M->sFngBX
z9XSSanT&Y3$aFTgwmi=gdRWZJ?7gj^*_*UDxzLOXJ`(DxF*(QV9TbN={#~?`mg&n8
z$UfjgVq)I)?dsq%`sCCZ2P%)I)h4A5Pu@VzZykLa$LOBb
z!6{Rqnh!PHk_`}l26n2U3x!~<&Cm|uhGng*9OkIxqaZk?Kfx5y7^Xl`CB*nm(R{S`}o7NbI(ZI@gW4BoqHBs
z+YWH;#Apy}qnVu-q;;_dN-G>yDhnm`P&qi;=qw#tGj%9h0Mut`W~Wi|(dboYDlZ6u
z-+}tWghMUw3u!IylOd%3Us8`r(tR@0W1^JL*Bs@)oiatbI(hOH!KLMGp=ZmDXrZUR
zcgrKZ6`ALjgCpdddSknX&=3-reDxZ2??8QXpu8c!si{f&1@GE>LvO`D#4`gV2XO-O
zQA4m#fTz}}I2ns>14<*qLhZb5dZc?eJ1`-tVQd=Q3_;lc>MbT<7=Q5|N1Ft0o9FpM
z#iNsai{@9?y)`Zg89@zeu8eKUnmcId>=duOc|#lC9iN;$=5^^_aczWG_XWf2-yVy*
zcGs65T-C2BVbHkv@;O;P1#_$F-W{(^XxLagHlidhBr87J%QIu(tkU8+6KYJv$W6LvarCxZ%p8O)M-6p$`e8|
zVxv4gv&!d`b{p3-7%-|5f98LZoq%+7rJh1dOFI=<7WgN4N1N=w&TY=C%r+d^VKy6O
z15D;ySF;%>aCnD$ZlmWGsc#Y
z2M%Ino_bR*aOb}e3sRpn;i8Kp4UUMuKArq{;q21gYlf}s)ob0bHG7s0EJ^Jj9Na&(
zq)bm3eSGz5Ny)2UF?ty_;y5A8mJ#ydh!G!ROOR!f;rIx?_l6CX?~WQZ>fOrq8wT}G
z9g{mcZ%k_M?`Pe9yS)7E+p}ihd~4vqw{FhfMC?zG9C=#0ziE?n|BOy|hS*d2xs2Z<
z*aO>K@orG}yT^F(ii_>KM0%j_E;@YP!`#A>*ftTPaityF@>-^iMtZBU_3!
zIeLs?Dj6>v7Q7k%cie|01f=6pH!Dq6Fy*DnC=nlSIVL1G3Eo?`=1NHWd
zW41j(ZE5gpM{3)9wQtu^1P3N|U2Sb6Ba3b9>~e~objdkv@@Z-xn|ms%r;Hb`NuJ-QbE)h8y
z-ChJ?2)9_~Dg?`Dp2|}>xk9kMLiRemU_3WNILc1}M%h94x7+JvbJR^_sPrkceh{Zo
zXqf`p(F}0_6cEjf%+VPcy7Y8iM#kvO$@x)HIXO{L`QkC;bJPE0GBU9>=PK@qOn
z_}r|G*|W>~H>QX5j^81)Z>%MD{;82Jp-FkM*<-WfhUP?;lteBCbdvcc{5rJe2L2ML
zwdv4~Z~xaxcjAaR&L`V<6Nwrbb_O<1J%oEsGJcdoYd-QlXB^&7L4)C
zB%Vh0g~+Bi9lbNU_MTmqH?CKQ&ZB)23xfNN(z@$24jiEVzDIgqcXUyAb+2^dT{At?
zJ$mq5e)%Im+Rt=!A_Y>{hEwjNp)=#c81xm+T?QhGh2BES!9JU=j2wAo6LH`zCVV+(
z_Ls={!`tlnS~5Y%fouVWTY`-|EmM}q88gb`_)9FMJQ?zt8!hFN?TGL=aSr>ets0$p
z4kk@_IK!Y(8ig^1+E>k_HM{_;XW7IJROgM%RRre-`DDdJTIKZ#$&dCN?i}dr?dOE>
z6gP=qM}EQ>x65FOo_G51aXZvD5IzY}~zDP=n^^8W#I~bkT~S
zQ_^@m6;dyW85HJb(u*i?f<|Nff}
z6egC(#FQr%CeQ=6PU32~HNpg8FSznDDDVoh3_C;8>(~WEwp8|xs3zA9m-*G4nk>i8
zmljCrh~qD?I|^{eNp{C_tb?||9oG&2!kr58jo``dXuusm$_}B9<>LmY1@#=9kuz8mI3Q!>
z>wWugA3tE-tT^BNzGQ^iG{O2+?60Eei+L2$dmGLk`vJ7$!nEx;S62&#vy%e8)-EBgakGyUA|qeccleCJH|K
z`TWQFPO|ooZUINickGbnu3jx`N5jnerK?x-BM@{1jYs;DDAspXS%X;a^FT9KkRNBKR0?h)1QX2_pk1cfjkJJVzrH#@tl3Y70F>zL{bcZDgEf{fh(!`@9R`r;ip7~M%D40e!hshn0?*TuH
zZ~G_<_}ndsu03^vp*^Gi2mxgd%EA>!!C_ub)*6=ptf&i`F?6W$B>h0_kR^TVwpFUb
z)tN(vW)6C{A!_^}cYohmncXV_=@tJd|M1??8I=)Y-^c$D5uvS*$zCzKV0dA7O?01v
z{%%p^rur
z?wXn5+?f-)x@Nd$*m=RbnU|Dg4meZS4Xh}ZUEPYaPvXvM^AH`7naV5?N{7f-HtLJU
zym?{{1-i17NE!+>#gd>sG1nlA1>OrQ&*(NH;Xu;J4dr9s8?-=r+-Gb;(16USkptsN
zdHXT@r|LEp#g6>P2>$w{e>SGl#U8b^XrU^JVWd@;&Q4kV*d
zXioODOJ^1*jjM=DtXtN5@J?cJc#?;-bTFyI>}PRrV<*moIHMYz(|$K6
zta}xxkM8P;aQhQxov4jzg@`GfG(cPkS23u?oy;-INb^y!KX8D=g=)2-!O6)jQtMhQ
zYHv-zin1m~m&S}TB#t4Hz;TE*LJu;gMHzWXYlO5FaDNTO2w&-kPU2VTIb%qsF+IKU
zZ$dI+)1#v@t-kMz1ZD(o1>!PYnC?R}-w@jku$C8EHWd~NQC$fdTNXyX~6C5j9CJj
z@HTk@Gzvu0=h#sL2dl}92pMNT%4acJX;ZsZjSd?Qkgt!$vf
z*;<*>pD2@lqM1VkQZJr2ifrDa6=DK%aF2=ukBN&HFP3&ZlZQ)Vr@fdD8nIJzY#FFU
zC#+YKF^J6cS9C|)>Z?XSY!nJ>q&!>VA#)s;B~!QZaBl(=&=}0^*(6NU|NEy7`^Z9f
zsc;s!=xI;#Wvzivw!9Tb$BLn{kum|n)VCCsjEQY;U5OYKNbldyH-p^1VU
z4an1Hwo19;&nDI8QEN+g_bI!3I
zORK%2Je3Zqc7cfg-5#|&Fj-gE9iP5vuYUXh+k`%GnEqPJIK
zpIBszbTWm4<@~5)i@JwaZLV+8H>@vESZ$EY{fiR?F1IGJLtOQ2coMNLTww{F+PBpg
zs?oZK1PA+tchT^=u4+FEoyyVC#my4yGXy{o`o05J1_37kD;mmR+C&OgsHk9y(G>@#
z5LdK{CI}<^Kt(?#EIPAhg^pjIzcMX#MUP(dDl`$5^Llk(k&?DNfAX2-1qI7K6@PA<
zsxSM<)?%ZLT`bZxrL%Q!jEuFj*{D$V+4S{H5<2tiO?@6+G-F4bT>c7-ii2$T=d^eu
zy;|#p)wx*-NYF_ravBh|2+s*S_-J_QlwBi7?U|C2GG(_;w|h$J!A?01
z`S}ewojP?JnV;8?+ma+FYfU>%4q?JlhkUwNh!NVDUf9
zJ4IdN2UsK0E}tQB2j57I$=lfq7ZqQAI)6!(R|M9)WGDj?z2ADPaP`nmS)E**G#>HY
z{SYO7{Fss1a?pL>L{YX#hwLdE%cU||$beKGs0Vux$TrAhkUQfIjTsucOfK)*mBY^d
zx=4EpJ|?dmV$EJx7RifxG3!En_-4;-Q+t$l6`r;)_ARcCG};$5%N+K_q%3M*$YZAM
z8#qThQSbXdSQu23`9VA=zK0QbAqxOeZ$GH|*Z{}ARgVEY;V(A!{6%ybvGaqPDWn~(Mu9vYbyOlK9ZYu^Uez|<%G?QG`t
zek@jRGNffwHx}<-RXs9L8vb;;ww!)`B+>_=;4Kbq1(
zOfGe1=fDC`Gs}mu1EXC8OWDK@uoVi&_~rCB7{3(U^qZq}eG{s>byer-a_d%x=Y%(j
z1&kYxJpt2%al_Sa*)MfZ81`~OV8P476B#!|pSFv)1f(#-58=EYd*O>jXtba%yl@)*
z!c;q{eANoyNGIOzkF_gTtbTyBUj4Z}5M}40zgcACWgI=q7O?&Q9-)wtL_uhSB#DC-=Fp8C*%r#056^8
zF2Ry6%Ed&ru|pQM26_6ndSiHQg1DDg%L|d8hc^@KaTOaKtc^{xhfP)tHDlU3rYFQ!
zBo3I&J_e;|G%2*mrp1-TOza(38acXeU{*$ONLCgdS-8}lECc^ip>_+qMwW%Zl+`Mj
zy2dDpDVv6}P!iECpnJG)fOmwQ(k(x}us%{38{07>KhV32lf^_kkBYLWVU?&Aepew`
zb_aciyhby`%28|Qp6w`Ww8i#6t`vII&xLS3c(>8v<;uXwu*ej$VJ0;H!IZd)L``Z$
ztiONkl-SCIfRwnTF8*<{JJvZP5~2
zk8Nztl_&ZUc-Od{>AqH9bqac|AMwOlZ7{UV$9>&pcSHsEdJAI59MPAeg=O;5@P63u
zb>_K7*xBM*-bLT^=W1UM<*pxvM;!EQYy<
zXZYu3SO6X^YVoE)F8k30ONG!#H$egQuv%Gel13uiq;uaAY30Ln;!R>DJtW}CL?0t<
zm7EM?+WMn2+CWerW5)D{<|roN4v0UI_3~u2X0?k08k#-z!hxuFN8`5wiF(3x;?Kei
z`8=FaOCQ-YPHt|t6j;c_peW^j#^j~KM9bPV**2_7-V6QbU@6wz2R_OiW|d7N7SbPn
z&qP?k4NU%ZaD(xT8|;E56bj2j2EL1zrszcGk2AU%pcrEbVk*}3ZKUha(9@2v5=Z0A
zxH=kk6MdmH%tIR+8Be1>M&*U=y#endvZJCkL`@x~y#D41zIp=LA@TBweERCg7i3)?
zU1X+7Xyc;-EJQ)=HIU(SpAw^A6SarXm5fh$HcdKQ3hI=-kZ)5ss&5gJUP$v00#>c+
z{NhRnTa|@(o3yTeLdpRJM(YA`_~d}Jt#IH^Nzvzcrl
z9pF(_>HBl;y)%>Ed+(Fpd+&`@QV1cW0U?znLqg~T2uSac01^oTBB(S46A+0A>RPs`
zt7}Q<#?&oOUt5%82&gQvUSBv)p$RSNQ#~w~ygp7pvvLzM=&dMM+`S
zs4!VQD~TIyEwkFmbxpWb18c#y81u-+JTkTOFq0uS#)L-OP>*gz(>G8UifyKpDSd?x
z1A8IaPqfU4L`4Xh6NUWxiF$60=CvO3x#-$6JUlG77#`L{+iAjgax=J`ofrogA&i!!
zY!{4UWo?a$B0aUMi;IDwFSUB#X7t3qVjO0*MIwSP;h87|Zz(7Th5+rIWmpir7a#uQ
zx_1xG+(XFI&#d{?!SY?=gn0JE`qSm1bvx^xTXVW1q<#;v9zR#S`;BGC{aJH{%XH&G$Kppnrq}7-u-4$5nRGY9;Dvh1=)XoG2SCri0xt6pf
zDCz>HLG4N@nUc+o&PS+!<%syVv(9Ti7_PdW(7I=q^NP`h;$^J5=DW*xH_X`c-e%53
z^WD5dtZXOXFZlP8o=B{ZKw?oDCx{-e_=y@$-XyNu6lc4
zY2Lua*7S}dfAK@|#hvryuiX1Ry%&}xB`v$q+tYt*dztZe<25(GU8COd)zdXKPk*&T
zz53gmYfQGAlx@GIc^XHVp5^&K;a&>Qf(@*(G}O`=2P+~I5}Yjz)xNsyZp;|vVu+71
zvz5QEP6iHYZJi7Pk~$Z(&UBw2+cw-(w5mSSU*9mXY2Up0N7`e@+%no`C7-
zwdNJHWx8>ZM~rNp%?!+ywyue8A^DMxcMn$t&t7w^p={~Pn{o@cz5JMU;l`%8
z#DxPn>3Lf^(=wMFCLd0G6CCeuEO1+Fj67#7tKfX8R2>ofg(B>T6Dq)5Qca>{J}Q-y
zQ>Zg(K2_l~o
zwF)krPimfCQoK5Y1n-?0R~~E?UY)b^>Rfm!n75*4u!^M34NdmSgOj2&&(puCExkaU
z?=5^kXG5-|^AqO&?n!=D!J%PxVMl)Rm`(Ti&Jxrrb=&m4vd&uWP|~cKZBw|ZB{8vO
zQ=v`j%GNyNq}I)NCt%t0e?^w0D>^d47b$a3PUd-=r~swf2&G@rTR|vEyq;n(NsNeX
z>{P%Wxx~G+9vWTNAbZ6K&iLcE%evz|v*x9gF3otcXx&IzXf&+8SUPMNuDX8PSLZ%s@9hI29_@?$ZW~=&i0?t7*p6=
z>=#(RBp;!akQx4-2?2I^mkzO?e-*sU7dvE~s&CB&BJ)9VFH+YCBU@`iRb_067CeMy
zJ0cXMV#)J}*b;CLzN=guDQt_e@JaGtm*1O_Q0mp_8er>|5nVm2FW)_O#eDG)?>zB^
z{M=HpvgX*SrqJvN4}-=OdyDh0f7(ahMK%caM>c5v
zkxqzf8rji$83gX&k~Bv=oyP4TQ9+qgy^%OIRMVFHkCsL8olnfC{>M@3b1d%4ai8XM
z^sPP8d*_TWGk<=9LqhXo)c3jqR7<@^(C>k<jza9RBrVqmR8i+#C_nJp3+x{Pp1lVS~v_pX-i^sotk9
z^u067zohp-wMtceptr=I+B!Gx{*x=v+RF+$$*J}-<2I9p12@K|77b(~-~G2Sv6lM$
z0i{U{3I;ft4{(bUpdAayigrl_286&1oSR&xHbq)C%XnwOGnVrXb;g!J{kA2{q|ptc
zB0Sv$_|U+?9K{dbCMVX3qp>p+d`-w16DJQleQTw{Fu*oxR=T_R&Jt7i02e(MePWu(
zt0q1aij2rJcEPE>cTT&j%n2D1^qgnS;_Zm5PoO0sJLF1A8%^R8{xtT<1sV!sd#>iD
zrta<$az`cxY1)x=29a4lc?juwghbZH_juj$1b!r?Ze%4egF;8#$sE7k!FxubZ_CCM
z@1m8nw(nj%wlO!e|3YU-ZN*!=x34@~Q_`Q;IdgF`NhtMfa|v~F&4`Vkcf2pH{NOL;
zNA?+*8AMfthn2>=_C!=K%|^XSuz`iCg`y5Kb`8tOPBBe^OrzA_zdrz$u{022I@o{Zr|UGvE%fIDMmq?mI7%
zd1GAu{TqJpAmh{1wjDII2T?UQ$mkGy)?EA)s6(N$Sqi5?qX4g#;^R};v&w}Fo5Gfv
zXPw`Zy>etu&$GSp(XEH)H>1?a5Uj;22c=EzS?(_Btp(ZhlAQ_ilD=4bV<4mA@Voui
z>X)|_WUjrmn0R}ktSN56#zN~^hnHk#b*)dDaK+pfh+FvIV{SpnfOgcEGw01)j73rD
zDe+Hw@~{N=?Io)$NancYW!E~+MR%0aBFy69CB>`qNPMYhqpQD-TV~9W54uv8ofC>`
z&aTSnNM%*N)jONrtb;0MMyeZ}%RQrGRW3dbMxL90v5l1j(;cGnZ6$WdP8KW+r+T%&
zA{<^^E3F)$>&0koxlRCX11<$5t0)}8CcH)?L7
zcCCASR|)SOWD>CT`8J#RFYYRzUcWYDQ)^;k>n8G=RFeUu3;wr^j%bywf!x5H(hv4l
zn;N2}pa-aj|LsH{sTp++g-T=Y7hDxm&=D`hA2=}aZgjl0nZbH@%fx|p?949oUimkk
z??P9=Olc3!W6vHgQH)kugV+X^6{@KwqwKpzU1IVB@;hhvaM=+zTq51#mRwjqXYaxo
zAzsr}(w*%#WA(wpO+>yT`SF*!7XSLmx~PuRy%?`b^pI0d$Kulga=AU~WjstVBF(U3
z7AKVnELJdOH2h987L=0a7|m2ks90P_O81O;XEzpPAN>C8nZNDJj8~V>PssP=gPWdS
z@$~DniSLX>d2SvUGAesPY{~AMJ)LhpQB!yP*NeNZABh`??B9|Jy{ur>wZ^hHI32
z#=d`^+x~}Rm7Fk&z^G`q=6pwZLI2^Rjf7jCbofeV@4F}Jau0kxLjLsQHyD38GDaNO
z_$lBUM#R>@z%SZWp^nyW3$+fgMpHLKYot)^%7^ogfElmN1!Crl30NrP8`eDpB^0$3viw@5WtKT|X)41=gmei%ET1X)$p0C}Qle(da8;&iFviB~_
zvS>KDz)Z&9MGB_SDSrvv!UlEduoBGl^<8~N>dt<;s&nI`ICEAS)^xztGD24X6@r