Skip to content

Consolidate support for Phi-1, Phi-1.5, and Phi-2 models #4552

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 16 commits into from
Closed

Consolidate support for Phi-1, Phi-1.5, and Phi-2 models #4552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
41a98c6
WIP: Initial setup for GGUF writer configuration
teleprint-me Nov 3, 2023
1aa3392
Merge branch 'master' into phi-1
teleprint-me Dec 13, 2023
748f376
fix: Apply phi to merged updates
teleprint-me Dec 13, 2023
1d4bcd2
Merge branch 'master' into phi-1
teleprint-me Dec 20, 2023
e53d44c
Consolidate PHI and PHI2 architectures in gguf constants
teleprint-me Dec 20, 2023
e96f40b
Update tensor mappings for Phi models (Phi-1, Phi-1.5, Phi-2)
teleprint-me Dec 20, 2023
ea6ae8d
Consolidate Phi model conversion handling in convert-hf-to-gguf.py
teleprint-me Dec 20, 2023
e290792
Consolidate Handling of Phi Models in llama.cpp
teleprint-me Dec 20, 2023
6becb1f
Remove deprecated conversion script
teleprint-me Dec 20, 2023
af9cd93
Ignore local content
teleprint-me Dec 21, 2023
348d565
Merge branch 'master' into phi-1
teleprint-me Dec 24, 2023
b0583f7
Merge branch 'master' into phi-1
teleprint-me Dec 25, 2023
5d546a3
Merge branch 'master' into phi-1
teleprint-me Dec 28, 2023
1a108ad
Merge branch 'master' into phi-1
teleprint-me Jan 7, 2024
46536a4
Merge branch 'master' into phi-1
teleprint-me Jan 9, 2024
2f2b3e4
chore: Apply flake8 formatting rules
teleprint-me Jan 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ build*/
out/
tmp/

local/
models/*
models-mnt

Expand Down
12 changes: 6 additions & 6 deletions convert-hf-to-gguf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import sys
from enum import IntEnum
from pathlib import Path
from typing import TYPE_CHECKING, Any, ContextManager, Iterator, cast, Optional
from typing import TYPE_CHECKING, Any, ContextManager, Iterator, Optional, cast

import numpy as np
import torch
Expand All @@ -22,9 +22,9 @@
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
import gguf


###### MODEL DEFINITIONS ######


class SentencePieceTokenTypes(IntEnum):
NORMAL = 1
UNKNOWN = 2
Expand Down Expand Up @@ -185,7 +185,7 @@ def from_model_architecture(model_architecture):
if model_architecture == "GPT2LMHeadModel":
return GPT2Model
if model_architecture == "PhiForCausalLM":
return Phi2Model
return PhiModel
if model_architecture == "PlamoForCausalLM":
return PlamoModel
return Model
Expand Down Expand Up @@ -230,7 +230,7 @@ def _get_model_architecture(self) -> gguf.MODEL_ARCH:
if arch == "GPT2LMHeadModel":
return gguf.MODEL_ARCH.GPT2
if arch == "PhiForCausalLM":
return gguf.MODEL_ARCH.PHI2
return gguf.MODEL_ARCH.PHI
if arch == "PlamoForCausalLM":
return gguf.MODEL_ARCH.PLAMO

Expand Down Expand Up @@ -1059,11 +1059,11 @@ def write_tensors(self):
self.gguf_writer.add_tensor("output.weight", data)


class Phi2Model(Model):
class PhiModel(Model):
def set_gguf_parameters(self):
block_count = self.hparams["n_layer"]

self.gguf_writer.add_name("Phi2")
self.gguf_writer.add_name("Phi")
self.gguf_writer.add_context_length(self.hparams["n_positions"])
self.gguf_writer.add_embedding_length(self.hparams["n_embd"])
self.gguf_writer.add_feed_forward_length(4 * self.hparams["n_embd"])
Expand Down
6 changes: 3 additions & 3 deletions gguf-py/gguf/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ class MODEL_ARCH(IntEnum):
BLOOM = auto()
STABLELM = auto()
QWEN = auto()
PHI2 = auto()
PHI = auto()
PLAMO = auto()


Expand Down Expand Up @@ -145,7 +145,7 @@ class MODEL_TENSOR(IntEnum):
MODEL_ARCH.BLOOM: "bloom",
MODEL_ARCH.STABLELM: "stablelm",
MODEL_ARCH.QWEN: "qwen",
MODEL_ARCH.PHI2: "phi2",
MODEL_ARCH.PHI: "phi",
MODEL_ARCH.PLAMO: "plamo",
}

Expand Down Expand Up @@ -383,7 +383,7 @@ class MODEL_TENSOR(IntEnum):
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
],
MODEL_ARCH.PHI2: [
MODEL_ARCH.PHI: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
MODEL_TENSOR.OUTPUT,
Expand Down
16 changes: 8 additions & 8 deletions gguf-py/gguf/tensor_mapping.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ class TensorNameMap:
"tok_embeddings", # llama-pth
"embeddings.word_embeddings", # bert
"language_model.embedding.word_embeddings", # persimmon
"transformer.embd.wte", # phi1 phi1_5 phi2
"wte", # gpt2
"transformer.embd.wte", # phi2
),

# Token type embeddings
Expand All @@ -44,7 +44,7 @@ class TensorNameMap:
"lm_head", # gpt2 mpt falcon llama-hf baichuan qwen
"output", # llama-pth bloom
"word_embeddings_for_head", # persimmon
"lm_head.linear", # phi2
"lm_head.linear", # phi1 phi1_5 phi2
),

# Output norm
Expand All @@ -57,7 +57,7 @@ class TensorNameMap:
"transformer.norm_f", # mpt
"ln_f", # refact bloom qwen gpt2
"language_model.encoder.final_layernorm", # persimmon
"lm_head.ln", # phi2
"lm_head.ln", # phi1 phi1_5 phi2
),

# Rope frequencies
Expand All @@ -80,8 +80,8 @@ class TensorNameMap:
"encoder.layer.{bid}.attention.output.LayerNorm", # bert
"language_model.encoder.layers.{bid}.input_layernorm", # persimmon
"model.layers.{bid}.ln1", # yi
"transformer.h.{bid}.ln", # phi1 phi1_5 phi2
"h.{bid}.ln_1", # gpt2
"transformer.h.{bid}.ln", # phi2
"model.layers.layers.{bid}.norm", # plamo
),

Expand All @@ -98,8 +98,8 @@ class TensorNameMap:
"transformer.h.{bid}.self_attention.query_key_value", # falcon
"h.{bid}.self_attention.query_key_value", # bloom
"language_model.encoder.layers.{bid}.self_attention.query_key_value", # persimmon
"transformer.h.{bid}.mixer.Wqkv", # phi1 phi1_5 phi2
"h.{bid}.attn.c_attn", # gpt2
"transformer.h.{bid}.mixer.Wqkv", # phi2
),

# Attention query
Expand Down Expand Up @@ -141,8 +141,8 @@ class TensorNameMap:
"encoder.layer.{bid}.attention.output.dense", # bert
"transformer.h.{bid}.attn.out_proj", # gpt-j
"language_model.encoder.layers.{bid}.self_attention.dense", # persimmon
"transformer.h.{bid}.mixer.out_proj", # phi1 phi1_5 phi2
"h.{bid}.attn.c_proj", # gpt2
"transformer.h.{bid}.mixer.out_proj", # phi2
"model.layers.layers.{bid}.self_attn.o_proj", # plamo
),

Expand Down Expand Up @@ -185,8 +185,8 @@ class TensorNameMap:
"transformer.h.{bid}.mlp.fc_in", # gpt-j
"language_model.encoder.layers.{bid}.mlp.dense_h_to_4h", # persimmon
"transformer.h.{bid}.mlp.w1", # qwen
"transformer.h.{bid}.mlp.fc1", # phi1 phi1_5 phi2
"h.{bid}.mlp.c_fc", # gpt2
"transformer.h.{bid}.mlp.fc1", # phi2
"model.layers.layers.{bid}.mlp.up_proj", # plamo
),

Expand Down Expand Up @@ -225,8 +225,8 @@ class TensorNameMap:
"encoder.layer.{bid}.output.dense", # bert
"transformer.h.{bid}.mlp.fc_out", # gpt-j
"language_model.encoder.layers.{bid}.mlp.dense_4h_to_h", # persimmon
"transformer.h.{bid}.mlp.fc2", # phi1 phi1_5 phi2
"h.{bid}.mlp.c_proj", # gpt2
"transformer.h.{bid}.mlp.fc2", # phi2
"model.layers.layers.{bid}.mlp.down_proj", # plamo
),

Expand Down
19 changes: 10 additions & 9 deletions llama.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ enum llm_arch {
LLM_ARCH_BLOOM,
LLM_ARCH_STABLELM,
LLM_ARCH_QWEN,
LLM_ARCH_PHI2,
LLM_ARCH_PHI,
LLM_ARCH_PLAMO,
LLM_ARCH_UNKNOWN,
};
Expand All @@ -216,7 +216,7 @@ static std::map<llm_arch, std::string> LLM_ARCH_NAMES = {
{ LLM_ARCH_BLOOM, "bloom" },
{ LLM_ARCH_STABLELM, "stablelm" },
{ LLM_ARCH_QWEN, "qwen" },
{ LLM_ARCH_PHI2, "phi2" },
{ LLM_ARCH_PHI, "phi" },
{ LLM_ARCH_PLAMO, "plamo" },
};

Expand Down Expand Up @@ -572,7 +572,7 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES =
},
},
{
LLM_ARCH_PHI2,
LLM_ARCH_PHI,
{
{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },
{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },
Expand Down Expand Up @@ -2824,11 +2824,12 @@ static void llm_load_hparams(
default: model.type = e_model::MODEL_UNKNOWN;
}
} break;
case LLM_ARCH_PHI2:
case LLM_ARCH_PHI:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);

switch (hparams.n_layer) {
case 24: model.type = e_model::MODEL_1B; break;
case 32: model.type = e_model::MODEL_3B; break;
default: model.type = e_model::MODEL_UNKNOWN;
}
Expand Down Expand Up @@ -3715,7 +3716,7 @@ static bool llm_load_tensors(
layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);
}
} break;
case LLM_ARCH_PHI2:
case LLM_ARCH_PHI:
{
model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);

Expand Down Expand Up @@ -4329,7 +4330,7 @@ static struct ggml_tensor * llm_build_kqv(
struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);
cb(kq, "kq", il);

if (model.arch == LLM_ARCH_PHI2) {
if (model.arch == LLM_ARCH_PHI) {
// for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs
// ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847
ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
Expand Down Expand Up @@ -5741,7 +5742,7 @@ struct llm_build_context {

return gf;
}
struct ggml_cgraph * build_phi2() {
struct ggml_cgraph * build_phi() {
struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);

const int64_t n_embd_head = hparams.n_embd_head_v;
Expand Down Expand Up @@ -6570,9 +6571,9 @@ static struct ggml_cgraph * llama_build_graph(
{
result = llm.build_qwen();
} break;
case LLM_ARCH_PHI2:
case LLM_ARCH_PHI:
{
result = llm.build_phi2();
result = llm.build_phi();
} break;
case LLM_ARCH_PLAMO:
{
Expand Down