Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/test_image.png filter=lfs diff=lfs merge=lfs -text
+llm/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+vlm/decoder.onnx_data filter=lfs diff=lfs merge=lfs -text
+vlm/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,312 @@
+### LLM text generation python examples
+
+```python
+from transformers import AutoConfig, AutoTokenizer
+import onnxruntime
+import numpy as np
+
+# 1. Load config, processor, and model
+path_to_model = "./llm"
+config = AutoConfig.from_pretrained(path_to_model)
+tokenizer = AutoTokenizer.from_pretrained(path_to_model)
+decoder_session = onnxruntime.InferenceSession(f"{path_to_model}/model_q4f16.onnx")
+
+## Set config values
+num_key_value_heads = config.num_key_value_heads
+head_dim = config.head_dim
+num_hidden_layers = config.num_hidden_layers
+eos_token_id = 106 # 106 is for <end_of_turn>
+
+# 2. Prepare inputs
+## Create input messages
+messages = [
+  { "role": "system", "content": "You are a helpful assistant." },
+  { "role": "user", "content": "Write me a short poem about Machine Learning." },
+]
+
+## Apply tokenizer
+inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="np")
+
+## Prepare decoder inputs
+batch_size = inputs['input_ids'].shape[0]
+past_key_values = {
+    f'past_key_values.{layer}.{kv}': np.zeros([batch_size, num_key_value_heads, 0, head_dim], dtype=np.float32)
+    for layer in range(num_hidden_layers)
+    for kv in ('key', 'value')
+}
+input_ids = inputs['input_ids']
+position_ids = np.tile(np.arange(1, input_ids.shape[-1] + 1), (batch_size, 1))
+
+# 3. Generation loop
+max_new_tokens = 128
+generated_tokens = np.array([[]], dtype=np.int64)
+for i in range(max_new_tokens):
+  logits, *present_key_values = decoder_session.run(None, dict(
+      input_ids=input_ids,
+      position_ids=position_ids,
+      **past_key_values,
+  ))
+
+  ## Update values for next generation loop
+  input_ids = logits[:, -1].argmax(-1, keepdims=True)
+  position_ids = position_ids[:, -1:] + 1
+  for j, key in enumerate(past_key_values):
+    past_key_values[key] = present_key_values[j]
+
+  generated_tokens = np.concatenate([generated_tokens, input_ids], axis=-1)
+  if (input_ids == eos_token_id).all():
+    break
+
+  ## (Optional) Streaming
+  print(tokenizer.decode(input_ids[0]), end='', flush=True)
+print()
+
+# 4. Output result
+print(tokenizer.batch_decode(generated_tokens))
+```
+
+### VLM text generation python examples
+```python
+import argparse
+import requests
+import onnxruntime
+from transformers import AutoTokenizer
+import numpy as np
+import time
+from PIL import Image
+
+IMAGE_TOKEN_INDEX = 151646
+MAX_GEN_LEN = 128
+USE_SAMPLING = True
+
+print("Loading inference sessions...")
+load_start = time.time()
+
+image_emb_session = onnxruntime.InferenceSession("vlm/vision_encoder.onnx")
+text_emb_session = onnxruntime.InferenceSession("vlm/token_embed_model.onnx")
+decoding_session = onnxruntime.InferenceSession("vlm/decoder.onnx")
+
+
+load_end = time.time()
+print(f"Inference sessions are loaded. Loading takes {load_end-load_start:0.2f} sec")
+
+
+def main(args):
+    tokenizer = AutoTokenizer.from_pretrained("./vlm")
+    tokenizer.add_tokens(["<image>"], special_tokens=True)
+
+    query = args.input_text
+    prompt = f"<|im_start|>user\n<image>\n{query}<|im_end|>\n<|im_start|>assistant\n"
+    past_kv_values, first_token, input_token_len = prefill(args, tokenizer, prompt)
+
+    decode(args, tokenizer, past_kv_values, first_token, input_token_len)
+
+
+def process_image(image_path):
+    # Load image
+    if "https" in image_path:
+        image = Image.open(requests.get(image_path, stream=True).raw)
+    else:
+        image = Image.open(image_path)
+    crop_size = (224, 224)
+    do_center_crop = True
+    do_convert_rgb = True
+    do_normalize = True
+    do_rescale = True
+    do_resize = True
+    image_mean = [0.48145466, 0.4578275, 0.40821073]
+    image_std = [0.26862954, 0.26130258, 0.27577711]
+    rescale_factor = 0.00392156862745098  # 1/255
+    size = {"shortest_edge": 224}
+    resample = Image.BICUBIC  # resample = 3
+
+    # Convert to rgb
+    if do_convert_rgb:
+        image = image.convert("RGB")
+
+    # Resize image
+    if do_resize:
+        shortest_edge = min(image.size)
+        scale_factor = size["shortest_edge"] / shortest_edge
+        new_size = (int(image.width * scale_factor), int(image.height * scale_factor))
+        image = image.resize(new_size, resample=resample)
+
+    # Center Crop
+    if do_center_crop:
+        left = (image.width - crop_size[0]) / 2
+        top = (image.height - crop_size[1]) / 2
+        right = (image.width + crop_size[0]) / 2
+        bottom = (image.height + crop_size[1]) / 2
+        image = image.crop((left, top, right, bottom))
+
+    # Convert to image array
+    image_array = np.array(image).astype(np.float32)
+
+    # Rescale (0-255 to 0-1)
+    if do_rescale:
+        image_array = image_array * rescale_factor
+
+    # Normalize
+    if do_normalize:
+        image_array = (image_array - image_mean) / image_std
+
+    # (H, W, C) -> (C, H, W)
+    image_array = np.transpose(image_array, (2, 0, 1))
+
+    # add batch dim (1, C, H, W)
+    image_array = np.expand_dims(image_array, axis=0)
+
+    return image_array.astype(np.float32)
+
+
+def top_p_sampling(last_logits, top_p=0.99):
+    sorted_indices = np.argsort(-last_logits)
+    sorted_logits = last_logits[sorted_indices]
+
+    cumulative_probs = np.cumsum(np.exp(sorted_logits - np.max(sorted_logits)))
+    cumulative_probs /= cumulative_probs[-1]
+
+    cutoff_index = np.searchsorted(cumulative_probs, top_p, side="right")
+
+    probs = np.exp(sorted_logits[: cutoff_index + 1] - np.max(sorted_logits[: cutoff_index + 1]))
+    probs /= np.sum(probs)
+
+    next_token = np.random.choice(sorted_indices[: cutoff_index + 1], p=probs)
+
+    return next_token
+
+
+# Prefill step
+# Inputs
+## input_ids: [1, seq_len]
+## past_key_values: each layer needs key[1, 2, 0, kv_dim], value[1, 2, 0, kv_dim] => total 56 kv
+# Outputs
+## logits: [1, seq_len, 151936]
+## present: each layer returns key[1, 2, seq_len, kv_dim], value[1, 2, seq_len, kv_dim] => total 56 kv
+def prefill(args, tokenizer, input_prompt):
+    print("Running prefill step...")
+    prefill_start = time.time()
+
+    input_ids = tokenizer(input_prompt)["input_ids"]
+    image_token_pos = input_ids.index(IMAGE_TOKEN_INDEX)
+
+    pixel_value = process_image(args.image_path)
+
+    # Get image embedding & Project image embedding to text embedding space
+    image_emb_output = image_emb_session.run(None, {"pixel_values": pixel_value})
+    image_features_proj = image_emb_output[0]
+
+    # Get text embedding
+    text_emb_output = text_emb_session.run(None, {"input_ids": [input_ids]})
+    input_features = text_emb_output[0]
+
+    # Split text embedding
+    pre_image_text_emb = input_features[:, :image_token_pos, :]
+    post_image_text_emb = input_features[:, image_token_pos + 1 :, :]
+
+    # Merge text embedding and image embedding
+    hidden_states = np.concatenate((pre_image_text_emb, image_features_proj, post_image_text_emb), axis=1)
+    input_token_len = hidden_states.shape[1]
+
+    # Prepare inputs used in prefill step with dummy input for initial past kv value
+    prefill_input = {
+        "/model/embed_tokens/Gather_output_0": hidden_states,
+        "attention_mask": np.expand_dims(np.ones(input_token_len).astype(np.int64), axis=0),
+        "position_ids": np.expand_dims(np.arange(input_token_len), axis=0),
+    }
+    for i in range(24):
+        entities = ["key", "value"]
+        for entity in entities:
+            input_name = f"past_key_values.{i}.{entity}"
+            prefill_input[input_name] = np.random.rand(1, 2, 0, 64).astype(np.float32)
+
+    # Run prefill
+    prefill_outputs = decoding_session.run(None, prefill_input)
+
+    # Get past kv values for decode step
+    past_kv_values = prefill_outputs[1:]
+
+    # Get first token with top-p sampling
+    if USE_SAMPLING:
+        last_logits = prefill_outputs[0][0][-1]
+        next_token = top_p_sampling(last_logits)
+    else:
+        next_token = prefill_outputs[0].argmax(-1)[0][-1]
+
+    prefill_done = time.time()
+    print(f"Prefill step done. Throughtput: {input_token_len/(prefill_done - prefill_start):0.2f} token/sec")
+
+    return past_kv_values, next_token, input_token_len
+
+
+# Generation step
+# Inputs
+## input_ids: [1, 1]
+## past_key_values: each layer needs key[1, 2, past_seq_len, kv_dim], value[1, 2, past_seq_len, kv_dim] => total 56 kv
+# Outputs
+## logits: [1, 1, 151936]
+## present: each layer returns key[1, 2, seq_len, kv_dim], value[1, 2, seq_len, kv_dim] => total 56 kv
+def decode(args, tokenizer, past_kv_values, first_token, input_token_len):
+    print("Runing decode step...", end="\n\n")
+    decode_start = time.time()
+
+    generated_ids = [first_token]
+    next_token = first_token
+
+    for last_token_id in range(MAX_GEN_LEN):
+        embedding_output = text_emb_session.run(None, {"input_ids": [[next_token]]})
+
+        # Get new token's embedding
+        hidden_states = embedding_output[0]
+
+        # Prepare inputs for decoding step
+        decoding_input = {
+            "/model/embed_tokens/Gather_output_0": hidden_states.astype(np.float32),
+            "attention_mask": [[1]],
+            "position_ids": [[input_token_len]],
+        }
+        input_token_len += 1
+        for j in range(24):
+            for k in range(2):
+                if k == 0:
+                    input_name = f"past_key_values.{j}.key"
+                else:
+                    input_name = f"past_key_values.{j}.value"
+                decoding_input[input_name] = past_kv_values[2 * j + k].astype(np.float32)
+
+        # Run decoding
+        decoding_outputs = decoding_session.run(None, decoding_input)
+
+        # Save kv values for next step
+        past_kv_values = decoding_outputs[1:]
+
+        # Get next token with top_p sampling
+        last_logits = decoding_outputs[0][0][-1]
+
+        if USE_SAMPLING:
+            next_token = top_p_sampling(last_logits)
+        else:
+            next_token = decoding_outputs[0].argmax(-1)[0][-1]
+
+        if next_token == tokenizer.eos_token_id:
+            break
+
+        # Save generated token
+        generated_ids.append(next_token)
+
+    decode_done = time.time()
+    response = tokenizer.decode(generated_ids)
+    with open(args.output_path, 'w') as f:
+        f.write(response)
+    print(f"\nDecode step done. Throughtput: {last_token_id/(decode_done - decode_start):0.2f} token/sec")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_text", type="str", help="Input query for inference", default="Where do you think this image is from?")
+    parser.add_argument("--image_path", type="str", help="Local image path or image url", default="assets/test_image.png")
+    parser.add_argument("--output_path", type="str", help="Output path to save the response", required=True)
+    args = parser.parse_args()
+
+    main(args)
+```

llm/model_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a8cb5ab287f04050d29de31e47354f8868069c0dec8cab326376274a6a12508
+size 997769309

llm/special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

llm/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

vlm/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

vlm/decoder.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74fe27d6c2e5c3c0f6a94cb8e8e62dfae8ab59db5e4b468bad57686dec87fee3
+size 1344441

vlm/decoder.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ccdf2f606eb40209e3e3385eab6e60933356ede96ac03db234d98cb27bb7978
+size 1991847936

vlm/requirements.txt

@@ -0,0 +1,4 @@
+torch
+transformers
+pillow
+requests

vlm/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

vlm/token_embedding_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cdb44e5aacbd9e54986b200eab130a1345d3ddc919032476ff54e3de8e130f2
+size 271751663

vlm/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcfe42da0a4497e8b2b172c1f9f4ec423a46dc12907f4349c55025f670422ba9
+size 11418266

vlm/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

vlm/vision_encoder.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5de39329bac62e7c7000f39c602369c1bec8bc1c496bbe50eec76bbefba6b5e4
+size 321017807