LLM/airllm
1
0
Fork 0
mirror of https://github.com/0xSojalSec/airllm.git synced 2026-03-07 14:24:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

add Mixtral

Browse Source
This commit is contained in:
Yu Li
2023-12-20 22:35:37 -06:00
parent e9f1cdec05
commit d2182a1bf6
7 changed files with 741 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
air_llm/README.md
View File

@@ -6,6 +6,8 @@ AirLLM优化inference内存，4GB单卡GPU可以运行70B大语言模型推理
## Updates
[2023/12/20] v2.7: Support AirLLMMixtral.
[2023/12/20] v2.6: Added AutoModel, automatically detect model type, no need to provide model class to initialize model.
提供AuoModel，自动根据repo参数检测模型类型，自动初始化模型。
@@ -165,6 +167,7 @@ Example colabs here:
| 9 | jondurbin/airoboros-l2-70b-2.2.1 | ✅ | AirLLMLlama2 |
| 10 | chargoddard/Yi-34B-Llama | ✅ | AirLLMLlama2 |
| ？ | mistralai/Mistral-7B-Instruct-v0.1 | ✅ | AirLLMMistral |
| ？ | mistralai/Mixtral-8x7B-v0.1 | ✅ | AirLLMMixtral |
#### [opencompass leaderboard](https://opencompass.org.cn/leaderboard-llm) top models

1
air_llm/airllm/__init__.py
View File

@@ -4,6 +4,7 @@ from .airllm_qwen import AirLLMQWen
from .airllm_baichuan import AirLLMBaichuan
from .airllm_internlm import AirLLMInternLM
from .airllm_mistral import AirLLMMistral
from .airllm_mixtral import AirLLMMixtral
from .airllm_base import AirLLMBaseModel
from .auto_model import AutoModel
from .utils import split_and_save_layers

22
air_llm/airllm/airllm_mixtral.py Normal file
View File

@@ -0,0 +1,22 @@
from transformers import GenerationConfig
from .airllm_base import AirLLMBaseModel
class AirLLMMixtral(AirLLMBaseModel):
def __init__(self, *args, **kwargs):
super(AirLLMMixtral, self).__init__(*args, **kwargs)
def get_use_better_transformer(self):
return False
def get_generation_config(self):
return GenerationConfig()

10
air_llm/airllm/auto_model.py
View File

@@ -1,13 +1,6 @@
import importlib
from transformers import AutoConfig
from .airllm import AirLLMLlama2
from .airllm_mistral import AirLLMMistral
from .airllm_baichuan import AirLLMBaichuan
from .airllm_internlm import AirLLMInternLM
from .airllm_chatglm import AirLLMChatGLM
from .airllm_qwen import AirLLMQWen
class AutoModel:
def __init__(self):
@@ -33,6 +26,8 @@ class AutoModel:
return "airllm", "AirLLMInternLM"
elif "Mistral" in config.architectures[0]:
return "airllm", "AirLLMMistral"
elif "Mixtral" in config.architectures[0]:
return "airllm", "AirLLMMixtral"
elif "Llama" in config.architectures[0]:
return "airllm", "AirLLMLlama2"
else:
@@ -43,7 +38,6 @@ class AutoModel:
def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
module, cls = AutoModel.get_module_class(pretrained_model_name_or_path, *inputs, **kwargs)
module = importlib.import_module(module)
class_ = getattr(module, cls)
return class_(pretrained_model_name_or_path, *inputs, ** kwargs)

2
air_llm/setup.py
View File

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
setuptools.setup(
name="airllm",
version="2.6.2",
version="2.7",
author="Gavin Li",
author_email="gavinli@animaai.cloud",
description="AirLLM allows single 4GB GPU card to run 70B large language models without quantization, distillation or pruning.",

3
air_llm/tests/test_automodel.py
View File

@@ -20,7 +20,8 @@ class TestAutoModel(unittest.TestCase):
'internlm/internlm-chat-7b': 'AirLLMInternLM',
'THUDM/chatglm3-6b-base': 'AirLLMChatGLM',
'baichuan-inc/Baichuan2-7B-Base': 'AirLLMBaichuan',
'mistralai/Mistral-7B-Instruct-v0.1': 'AirLLMMistral'
'mistralai/Mistral-7B-Instruct-v0.1': 'AirLLMMistral',
'mistralai/Mixtral-8x7B-v0.1': 'AirLLMMixtral'
}

710
air_llm/tests/test_notebooks/test_mixtral.ipynb Normal file
View File

@@ -0,0 +1,710 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "440851a0-170d-4226-9857-f39f05cc6c70",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Looking in indexes: http://mirrors.tencentyun.com/pypi/simple\n",
"Requirement already satisfied: airllm in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (0.9.1)\n",
"Collecting airllm\n",
" Downloading http://mirrors.tencentyun.com/pypi/packages/b5/36/d1cefb0725097e7ddf907783f31e9e17b191009978839a3d06598e72c41d/airllm-2.6-py3-none-any.whl (33 kB)\n",
"Requirement already satisfied: transformers in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (4.35.0)\n",
"Collecting transformers\n",
" Downloading http://mirrors.tencentyun.com/pypi/packages/20/0a/739426a81f7635b422fbe6cb8d1d99d1235579a6ac8024c13d743efa6847/transformers-4.36.2-py3-none-any.whl (8.2 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.2/8.2 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m0m\n",
"\u001b[?25hRequirement already satisfied: tqdm in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from airllm) (4.66.1)\n",
"Requirement already satisfied: torch in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from airllm) (2.1.0)\n",
"Requirement already satisfied: accelerate in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from airllm) (0.24.1)\n",
"Requirement already satisfied: safetensors in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from airllm) (0.4.0)\n",
"Requirement already satisfied: optimum in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from airllm) (1.14.0)\n",
"Requirement already satisfied: huggingface-hub in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from airllm) (0.17.3)\n",
"Collecting scipy (from airllm)\n",
" Downloading http://mirrors.tencentyun.com/pypi/packages/69/f0/fb07a9548e48b687b8bf2fa81d71aba9cfc548d365046ca1c791e24db99d/scipy-1.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34.5 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m34.5/34.5 MB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: filelock in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from transformers) (3.9.0)\n",
"Collecting huggingface-hub (from airllm)\n",
" Downloading http://mirrors.tencentyun.com/pypi/packages/a0/0a/02ac0ae1047d97769003ff4fb8e6717024f3f174a5d13257415aa09e13d9/huggingface_hub-0.20.1-py3-none-any.whl (330 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m330.1/330.1 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: numpy>=1.17 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from transformers) (1.24.3)\n",
"Requirement already satisfied: packaging>=20.0 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from transformers) (23.2)\n",
"Requirement already satisfied: pyyaml>=5.1 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from transformers) (6.0.1)\n",
"Requirement already satisfied: regex!=2019.12.17 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from transformers) (2023.10.3)\n",
"Requirement already satisfied: requests in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from transformers) (2.31.0)\n",
"Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from transformers) (0.14.1)\n",
"Requirement already satisfied: fsspec>=2023.5.0 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from huggingface-hub->airllm) (2023.10.0)\n",
"Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from huggingface-hub->airllm) (4.7.1)\n",
"INFO: pip is looking at multiple versions of tokenizers to determine which version is compatible with other requirements. This could take a while.\n",
"Collecting tokenizers<0.19,>=0.14 (from transformers)\n",
" Downloading http://mirrors.tencentyun.com/pypi/packages/ad/75/56230c5c65b226e707e1adbc759c19fdf1b20bb02c0276796b132c97118a/tokenizers-0.15.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m0m\n",
"\u001b[?25hRequirement already satisfied: psutil in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from accelerate->airllm) (5.9.6)\n",
"Requirement already satisfied: sympy in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from torch->airllm) (1.11.1)\n",
"Requirement already satisfied: networkx in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from torch->airllm) (3.1)\n",
"Requirement already satisfied: jinja2 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from torch->airllm) (3.1.2)\n",
"Requirement already satisfied: coloredlogs in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from optimum->airllm) (15.0.1)\n",
"Requirement already satisfied: datasets in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from optimum->airllm) (2.14.6)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from requests->transformers) (2.0.4)\n",
"Requirement already satisfied: idna<4,>=2.5 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from requests->transformers) (3.4)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from requests->transformers) (1.26.18)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from requests->transformers) (2023.7.22)\n",
"Requirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from transformers[sentencepiece]>=4.26.0->optimum->airllm) (0.1.99)\n",
"Requirement already satisfied: protobuf in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from transformers[sentencepiece]>=4.26.0->optimum->airllm) (4.25.0)\n",
"Requirement already satisfied: humanfriendly>=9.1 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from coloredlogs->optimum->airllm) (10.0)\n",
"Requirement already satisfied: pyarrow>=8.0.0 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from datasets->optimum->airllm) (14.0.0)\n",
"Requirement already satisfied: dill<0.3.8,>=0.3.0 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from datasets->optimum->airllm) (0.3.7)\n",
"Requirement already satisfied: pandas in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from datasets->optimum->airllm) (2.0.3)\n",
"Requirement already satisfied: xxhash in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from datasets->optimum->airllm) (3.4.1)\n",
"Requirement already satisfied: multiprocess in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from datasets->optimum->airllm) (0.70.15)\n",
"Requirement already satisfied: aiohttp in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from datasets->optimum->airllm) (3.8.6)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from jinja2->torch->airllm) (2.1.1)\n",
"Requirement already satisfied: mpmath>=0.19 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from sympy->torch->airllm) (1.3.0)\n",
"Requirement already satisfied: attrs>=17.3.0 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from aiohttp->datasets->optimum->airllm) (23.1.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from aiohttp->datasets->optimum->airllm) (6.0.4)\n",
"Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from aiohttp->datasets->optimum->airllm) (4.0.3)\n",
"Requirement already satisfied: yarl<2.0,>=1.0 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from aiohttp->datasets->optimum->airllm) (1.9.2)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from aiohttp->datasets->optimum->airllm) (1.4.0)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from aiohttp->datasets->optimum->airllm) (1.3.1)\n",
"Requirement already satisfied: python-dateutil>=2.8.2 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from pandas->datasets->optimum->airllm) (2.8.2)\n",
"Requirement already satisfied: pytz>=2020.1 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from pandas->datasets->optimum->airllm) (2023.3.post1)\n",
"Requirement already satisfied: tzdata>=2022.1 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from pandas->datasets->optimum->airllm) (2023.3)\n",
"Requirement already satisfied: six>=1.5 in /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas->datasets->optimum->airllm) (1.16.0)\n",
"Installing collected packages: scipy, huggingface-hub, tokenizers, transformers, airllm\n",
" Attempting uninstall: huggingface-hub\n",
" Found existing installation: huggingface-hub 0.17.3\n",
" Uninstalling huggingface-hub-0.17.3:\n",
" Successfully uninstalled huggingface-hub-0.17.3\n",
" Attempting uninstall: tokenizers\n",
" Found existing installation: tokenizers 0.14.1\n",
" Uninstalling tokenizers-0.14.1:\n",
" Successfully uninstalled tokenizers-0.14.1\n",
" Attempting uninstall: transformers\n",
" Found existing installation: transformers 4.35.0\n",
" Uninstalling transformers-4.35.0:\n",
" Successfully uninstalled transformers-4.35.0\n",
" Attempting uninstall: airllm\n",
" Found existing installation: airllm 0.9.1\n",
" Uninstalling airllm-0.9.1:\n",
" Successfully uninstalled airllm-0.9.1\n",
"Successfully installed airllm-2.6 huggingface-hub-0.20.1 scipy-1.10.1 tokenizers-0.15.0 transformers-4.36.2\n"
]
}
],
"source": [
"!pip install -U airllm transformers"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "47de26bf-510a-4f8f-ae99-c4c22a0e12b7",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "d8829a3be8414a3d82aacee5f801ece7",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"config.json: 0%| | 0.00/720 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "aed2af7d16004b849a401749c74346c8",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model.safetensors.index.json: 0%| | 0.00/92.7k [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e68dab4287b3496ba453c2664e167d4e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading shards: 0%| | 0/19 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "93c15f55db52460cb5ff5bffd604876e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00001-of-00019.safetensors: 0%| | 0.00/4.89G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "02299832dca2482ba4e84b5a34b6604d",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00002-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "83dfc1d781dc4263958767984c60e5f3",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00003-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1838e6535eed4020bbc52404a5f0e40e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00004-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b0282171d02d43c883e0918139336e7b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00005-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "18a1e681ca7c40d78a2ecb7be9821548",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00006-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "da04e3e27395458b87c8320150cb5167",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00007-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "d3af708bd5984d4b8c6552afbbbf7767",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00008-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "05c6af1b24224cc6b7578e6ea8fbf0e0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00009-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "33b169a15603482281fd45ab12a1c1e7",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00010-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "010af99eef2b4955a798bcbac95b3789",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00011-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "627af607d99c421688698093888c9da6",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00012-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2cd9e5c547024ffca634e1a97f0b8933",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00013-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e8c7cd63a0794a668820c5f1d7bb90ca",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00014-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "d6154839ba9946fbbf27563d8c76ee77",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00015-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ad8792741a54426f8db778ea5203b6cc",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00016-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c72707d9eee54201a36c94e0a1ea0a79",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00017-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e8ed9408e2834081bd4f20af78d16bc0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00018-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0d3ee31f8667458bb9d23f0cf47f1c9e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00019-of-00019.safetensors: 0%| | 0.00/4.22G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "06963554ce7f459b89728330ac5cd589",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Loading checkpoint shards: 0%| | 0/19 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"KeyboardInterrupt\n",
"\n"
]
}
],
"source": [
"from transformers import AutoModel\n",
"\n",
"model = AutoModel.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "3d670170-b8fe-4d23-96bd-e9d4c1710e57",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "71376c9f04e042678de531b3e06ef9b7",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"tokenizer_config.json: 0%| | 0.00/967 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b946a8fa81a34ec6af8dfca856ff5788",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"tokenizer.model: 0%| | 0.00/493k [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8105390138fb4e12a7ceda5ab4a73dc0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"tokenizer.json: 0%| | 0.00/1.80M [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "3e7442f9702e4870b7f49f93be265b32",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"special_tokens_map.json: 0%| | 0.00/72.0 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from transformers import AutoTokenizer, AutoConfig\n",
"\n",
"t = AutoTokenizer.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")\n",
"c = AutoConfig.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "072d6f69-aa58-4d00-9fb9-fb397a8231b2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Name: airllm\n",
"Version: 2.6\n",
"Summary: AirLLM allows single 4GB GPU card to run 70B large language models without quantization, distillation or pruning.\n",
"Home-page: https://github.com/lyogavin/Anima/tree/main/air_llm\n",
"Author: Gavin Li\n",
"Author-email: gavinli@animaai.cloud\n",
"License: \n",
"Location: /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages\n",
"Requires: accelerate, huggingface-hub, optimum, safetensors, scipy, torch, tqdm, transformers\n",
"Required-by: \n"
]
}
],
"source": [
"!pip show airllm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "17ba0595-b712-49f1-aa7b-997e01118a92",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"../../__init__.py ../../airllm_mixtral.py ../../auto_model.py\n"
]
}
],
"source": [
"!ls ../../*.py"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a096444f-c3d9-457b-86bd-95084bf93d90",
"metadata": {},
"outputs": [],
"source": [
"!cp ../../*.py /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages/airllm/"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "a4c321ef-0575-43db-b947-880ae484189d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"--2023-12-21 12:17:37-- https://huggingface.co/mistralai/Mixtral-8x7B-v0.1/raw/main/generation_config.json\n",
"Resolving huggingface.co (huggingface.co)... 108.138.246.71, 108.138.246.85, 108.138.246.79, ...\n",
"Connecting to huggingface.co (huggingface.co)|108.138.246.71|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 116 [text/plain]\n",
"Saving to: ‘generation_config.json’\n",
"\n",
"generation_config.j 100%[===================>] 116 --.-KB/s in 0s \n",
"\n",
"2023-12-21 12:17:37 (27.1 MB/s) - ‘generation_config.json’ saved [116/116]\n",
"\n"
]
}
],
"source": [
"!wget https://huggingface.co/mistralai/Mixtral-8x7B-v0.1/raw/main/generation_config.json\n",
"!cp generation_config.json /home/ubuntu/.cache/huggingface/hub/models--mistralai--Mixtral-8x7B-v0.1/snapshots/58301445dc1378584211722b7ebf8743ec4e192b/"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "19eb8fee-ab17-4a54-9af2-ca809bd096b5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
">>>> cache_utils installed\n",
"saved layers already found in /home/ubuntu/.cache/huggingface/hub/models--mistralai--Mixtral-8x7B-v0.1/snapshots/58301445dc1378584211722b7ebf8743ec4e192b/splitted_model\n",
"either BetterTransformer or attn_implementation='sdpa' is available, creating model directly\n",
"either BetterTransformer or attn_implementation='sdpa' is available, creating model directly\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"cuda:0: 100%|██████████| 35/35 [04:29<00:00, 7.69s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"either BetterTransformer or attn_implementation='sdpa' is available, creating model directly\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"cuda:0: 100%|██████████| 35/35 [04:30<00:00, 7.73s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"either BetterTransformer or attn_implementation='sdpa' is available, creating model directly\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"cuda:0: 86%|████████▌ | 30/35 [04:03<00:41, 8.36s/it]"
]
}
],
"source": [
"from airllm import AutoModel\n",
"\n",
"MAX_LENGTH = 128\n",
"# could use hugging face model repo id:\n",
"model = AutoModel.from_pretrained(\"/home/ubuntu/.cache/huggingface/hub/models--mistralai--Mixtral-8x7B-v0.1/snapshots/58301445dc1378584211722b7ebf8743ec4e192b/\")\n",
"\n",
"input_text = [\n",
" 'I like',\n",
" ]\n",
"\n",
"input_tokens = model.tokenizer(input_text,\n",
" return_tensors=\"pt\",\n",
" return_attention_mask=False,\n",
" truncation=True,\n",
" max_length=MAX_LENGTH,\n",
" #padding=True\n",
" )\n",
"\n",
"generation_output = model.generate(\n",
" input_tokens['input_ids'].cuda(),\n",
" max_new_tokens=3,\n",
" use_cache=True,\n",
" return_dict_in_generate=True)\n",
"\n",
"model.tokenizer.decode(generation_output.sequences[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "db4d8871-7e30-4eb8-b2f9-0310409c71d7",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.18"
}
},
"nbformat": 4,
"nbformat_minor": 5
}