Upload 57.6M parameter custom language model

.ipynb_checkpoints/upload_to_hf-checkpoint.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+"""Automated Hugging Face Upload Script"""
+
+from huggingface_hub import HfApi, upload_folder
+
+def upload_model():
+    model_dir = "./hf_model"
+    repo_id = "your-username/custom-57m-language-model"
+
+    print("🤗 Starting Hugging Face upload...")
+    print(f"📁 Model directory: {model_dir}")
+    print(f"🔗 Repository: {repo_id}")
+
+    try:
+        api = HfApi()
+        api.create_repo(repo_id=repo_id, exist_ok=True)
+        print("✅ Repository created/verified")
+
+        upload_folder(
+            folder_path=model_dir,
+            repo_id=repo_id,
+            repo_type="model",
+            commit_message="Upload 57.6M parameter custom language model"
+        )
+
+        print("🎉 Upload completed!")
+        print(f"🔗 Your model: https://huggingface.co/{repo_id}")
+
+    except Exception as e:
+        print(f"❌ Upload failed: {e}")
+        print("💡 Make sure you're logged in: huggingface-cli login")
+
+if __name__ == "__main__":
+    upload_model()

README.md

@@ -0,0 +1,83 @@
+---
+language: en
+license: mit
+tags:
+- causal-lm
+- custom-architecture
+- transformer
+pipeline_tag: text-generation
+---
+
+# Custom 57M Language Model
+
+A custom 57.55M parameter causal language model with modern transformer architecture.
+
+## Model Details
+
+- **Parameters**: 57,553,632 (57.55M)
+- **Architecture**: 12-layer Transformer
+- **Hidden Size**: 432
+- **Attention Heads**: 8
+- **Head Dimension**: 54
+- **Intermediate Size**: 1,728
+- **Vocabulary Size**: 50,257 (GPT-2 tokenizer)
+- **Max Sequence Length**: 1,024
+
+## Architecture Features
+
+- **RoPE Positional Embeddings**: Rotary Position Embedding (θ=10000.0)
+- **SwiGLU Activation**: Swish-Gated Linear Unit in feed-forward networks
+- **RMSNorm**: Root Mean Square Layer Normalization (ε=1e-06)
+- **Tied Embeddings**: Input and output embeddings share weights
+- **Dropout**: 0.1 dropout rate
+
+## Training Configuration
+
+- **Dummy Phase**: 2 epochs, 1,000 samples, LR=0.0005
+- **C4 Phase**: 3 epochs, 1,000 samples, LR=0.0003
+- **Optimizer**: AdamW (weight_decay=0.1)
+- **Scheduler**: Cosine Annealing
+- **Gradient Clipping**: 1.0
+
+## Generation Parameters
+
+- **Temperature**: 0.8
+- **Top-K**: 50
+- **Top-P**: 0.9
+- **Repetition Penalty**: 1.1
+- **Max New Tokens**: 100
+
+## Usage
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+tokenizer = AutoTokenizer.from_pretrained("your-username/custom-57m-language-model")
+model = AutoModelForCausalLM.from_pretrained("your-username/custom-57m-language-model")
+
+input_text = "The future of artificial intelligence"
+inputs = tokenizer.encode(input_text, return_tensors='pt')
+outputs = model.generate(
+    inputs, 
+    max_length=100, 
+    temperature=0.8,
+    top_k=50,
+    top_p=0.9,
+    repetition_penalty=1.1
+)
+generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+print(generated_text)
+```
+
+## Training Dataset
+
+- **Primary**: C4 (Colossal Clean Crawled Corpus)
+- **Warm-up**: Synthetic dummy data for initial training
+
+## License
+
+MIT License
+
+## Model Card
+
+This model was trained as an educational demonstration of transformer architecture implementation with modern techniques like RoPE embeddings and SwiGLU activations.

config.json

@@ -0,0 +1,21 @@
+{
+  "architectures": [
+    "HuggingFaceCompatibleModel"
+  ],
+  "bos_token_id": 50256,
+  "dim": 432,
+  "dropout": 0.1,
+  "eos_token_id": 50256,
+  "head_dim": 54,
+  "intermediate_size": 1728,
+  "layer_norm_eps": 1e-06,
+  "max_seq_len": 1024,
+  "model_type": "custom_language_model",
+  "n_heads": 8,
+  "n_layers": 12,
+  "pad_token_id": 50256,
+  "rope_theta": 10000.0,
+  "torch_dtype": "float32",
+  "transformers_version": "4.52.4",
+  "vocab_size": 50257
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f70f07fe036168da923ce2d0901aa16765ca347fc5d60af753e4af8eef865f93
+size 230250779

requirements.txt

@@ -0,0 +1,4 @@
+torch>=2.0.0
+transformers>=4.30.0
+tokenizers>=0.13.0
+numpy>=1.24.0

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

upload_to_hf.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+"""Automated Hugging Face Upload Script"""
+
+from huggingface_hub import HfApi, upload_folder
+
+def upload_model():
+    model_dir = "./hf_model"
+    repo_id = "Mostafa8Mehrabi/custom-57m-language-model"
+
+    print("🤗 Starting Hugging Face upload...")
+    print(f"📁 Model directory: {model_dir}")
+    print(f"🔗 Repository: {repo_id}")
+
+    try:
+        api = HfApi()
+        api.create_repo(repo_id=repo_id, exist_ok=True)
+        print("✅ Repository created/verified")
+
+        upload_folder(
+            folder_path=model_dir,
+            repo_id=repo_id,
+            repo_type="model",
+            commit_message="Upload 57.6M parameter custom language model"
+        )
+
+        print("🎉 Upload completed!")
+        print(f"🔗 Your model: https://huggingface.co/{repo_id}")
+
+    except Exception as e:
+        print(f"❌ Upload failed: {e}")
+        print("💡 Make sure you're logged in: huggingface-cli login")
+
+if __name__ == "__main__":
+    upload_model()