Skip to content

Gguf: write tensors in a single pass #2644

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Aug 17, 2023
Merged

Gguf: write tensors in a single pass #2644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
f31e923
gguf : single pass for writing tensors + refactoring writer
monatis Aug 17, 2023
dce07c3
gguf : single pass for writing tensors + refactoring writer
monatis Aug 17, 2023
5f97a48
gguf : single pass for writing tensors + refactoring writer
monatis Aug 17, 2023
2f8fc92
gguf : fix conflicts
monatis Aug 17, 2023
22c61c5
gguf : style fixes in simple conversion script
monatis Aug 17, 2023
9f02694
gguf : refactor gptneox conversion script
monatis Aug 17, 2023
f717041
gguf : rename h5 to hf (for HuggingFace)
monatis Aug 17, 2023
1d93d04
gguf : refactor pth to gguf conversion script
monatis Aug 17, 2023
4dbce7d
gguf : rm file_type key and method
monatis Aug 17, 2023
e426b3c
gguf.py : fix vertical alignment
ggerganov Aug 17, 2023
307e09c
Merge branch 'gguf' into gguf-write-single-pass
ggerganov Aug 17, 2023
6a9e637
gguf.py : indentation
ggerganov Aug 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 34 additions & 83 deletions convert-gptneox-h5-to-gguf.py → convert-gptneox-hf-to-gguf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
from transformers import AutoTokenizer

# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py


def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
Expand All @@ -34,6 +36,7 @@ def bytes_to_unicode():
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))


def count_model_parts(dir_model: str) -> int:
num_parts = 0
for filename in os.listdir(dir_model):
Expand All @@ -44,6 +47,7 @@ def count_model_parts(dir_model: str) -> int:
print("gguf: found " + str(num_parts) + " model parts")
return num_parts


if len(sys.argv) < 3:
print("Usage: convert-h5-to-ggml.py dir-model ftype\n")
print(" ftype == 0 -> float32")
Expand All @@ -58,7 +62,7 @@ def count_model_parts(dir_model: str) -> int:
# possible tensor data types
# ftype == 0 -> float32
# ftype == 1 -> float16
#

# map from ftype to string
ftype_str = ["f32", "f16"]

Expand All @@ -67,6 +71,7 @@ def count_model_parts(dir_model: str) -> int:
ftype = int(sys.argv[2])
if ftype < 0 or ftype > 1:
print("Invalid ftype: " + str(ftype))

sys.exit(1)

fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf"
Expand All @@ -77,30 +82,30 @@ def count_model_parts(dir_model: str) -> int:
hparams = json.load(f)

if hparams["architectures"][0] != "GPTNeoXForCausalLM":
print("Model architecture not supported: " + hparams["architectures"][0] )
print("Model architecture not supported: " + hparams["architectures"][0])

sys.exit()

# get number of model parts
num_parts = count_model_parts(dir_model)

gguf_writer = gguf.GGUFWriter.open(fname_out)
llm_arch = "gptneox"
gguf_writer = gguf.GGUFWriter(fname_out, arch=llm_arch)

print("gguf: get model metadata")

llm_arch = "gptneox"
block_count = hparams["num_hidden_layers"]

gguf_writer.add_architecture(llm_arch)
gguf_writer.add_architecture()
gguf_writer.add_name(last_dir)
gguf_writer.add_file_type( "All tensors F32" if ftype == 0 else "Most tensors F16, some F32")
gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"])
gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"])
gguf_writer.add_block_count(llm_arch, block_count)
gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"])
gguf_writer.add_rope_dimension_count(llm_arch, int( hparams["rotary_pct"]*(hparams["hidden_size"]//hparams["num_attention_heads"])) )
gguf_writer.add_head_count(llm_arch, hparams["num_attention_heads"])
gguf_writer.add_parallel_residual(llm_arch, hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True)
gguf_writer.add_layer_norm_eps(llm_arch, hparams["layer_norm_eps"])
gguf_writer.add_context_length(hparams["max_position_embeddings"])
gguf_writer.add_embedding_length(hparams["hidden_size"])
gguf_writer.add_block_count(block_count)
gguf_writer.add_feed_forward_length(hparams["intermediate_size"])
gguf_writer.add_rope_dimension_count(int(hparams["rotary_pct"]*(hparams["hidden_size"]//hparams["num_attention_heads"])))
gguf_writer.add_head_count(hparams["num_attention_heads"])
gguf_writer.add_parallel_residual(hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True)
gguf_writer.add_layer_norm_eps(hparams["layer_norm_eps"])

# TOKENIZATION

Expand All @@ -124,14 +129,14 @@ def count_model_parts(dir_model: str) -> int:

print("gguf: get gpt2 tokenizer vocab")

vocab_size = len( tokenizer_json["model"]["vocab"] )
vocab_size = len(tokenizer_json["model"]["vocab"])

# ref: https://github.com/cmp-nct/ggllm.cpp/blob/master/falcon_convert.py
tokenizer = AutoTokenizer.from_pretrained(dir_model)

reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
byte_encoder = bytes_to_unicode()
byte_decoder = {v:k for k, v in byte_encoder.items()}
byte_decoder = {v: k for k, v in byte_encoder.items()}

for i in range(vocab_size):
if i in reverse_vocab:
Expand All @@ -146,8 +151,9 @@ def count_model_parts(dir_model: str) -> int:
text.extend(c.encode('utf-8'))
else:
print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
padding_token = f"[PAD{i}]".encode("utf8")
text = bytearray(padding_token)
pad_token = f"[PAD{i}]".encode("utf8")
text = bytearray(pad_token)

tokens.append(text)

gguf_writer.add_token_list(tokens)
Expand Down Expand Up @@ -201,7 +207,7 @@ def count_model_parts(dir_model: str) -> int:
)

for part_name in part_names:
print("gguf: loading model part '"+ part_name + "'")
print("gguf: loading model part '" + part_name + "'")
model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu")

for name in model_part.keys():
Expand All @@ -223,11 +229,12 @@ def count_model_parts(dir_model: str) -> int:
elif name.endswith(".bias") and name[:-5] in tensor_map:
name = tensor_map[name[:-5]] + ".bias"
else:
print( "Can not map tensor '" + name + "'" )
print("Can not map tensor '" + name + "'")
sys.exit()

n_dims = len(data.shape)
data_dtype = data.dtype
old_dtype = data_dtype

# if f32 desired, convert any float16 to float32
if ftype == 0 and data.dtype == np.float16:
Expand All @@ -241,77 +248,21 @@ def count_model_parts(dir_model: str) -> int:
if ftype == 1 and data.dtype == np.float32 and name.endswith(".weight") and n_dims == 2:
data_dtype = np.float16

data_nbytes = data.size * 2 if data_dtype == np.float16 else data.size * 4
print(name + ", n_dims = " + str(n_dims) + ", " + str(old_dtype) + " --> " + str(data_dtype))

data = data.astype(data_dtype)

gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes)
gguf_writer.add_tensor(name, data)


print("gguf: write header")
gguf_writer.write_header_to_file()
print("gguf: write metadata")
gguf_writer.write_kv_data_to_file()
print("gguf: write tensor metadata")
gguf_writer.write_ti_data_to_file()

# tensor data
print("gguf: convert and write tensor data")

if num_parts == 0:
part_names = ("pytorch_model.bin",)
else:
part_names = (
f"pytorch_model-{n:05}-of-{num_parts:05}.bin" for n in range(1, num_parts + 1)
)

for part_name in part_names:
print("gguf: loading model part '"+ part_name + "'")
model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu")

for name in model_part.keys():
data = model_part[name]

old_dtype = data.dtype

# we don't need these
if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"):
continue

# convert any unsupported data types to float32
if data.dtype != torch.float16 and data.dtype != torch.float32:
data = data.to(torch.float32)

data = data.squeeze().numpy()

# map tensor names
if name.endswith(".weight") and name[:-7] in tensor_map:
name = tensor_map[name[:-7]] + ".weight"
elif name.endswith(".bias") and name[:-5] in tensor_map:
name = tensor_map[name[:-5]] + ".bias"
else:
print( "Can not map tensor '" + name + "'" )
sys.exit()

n_dims = len(data.shape)
data_dtype = data.dtype

# if f32 desired, convert any float16 to float32
if ftype == 0 and data.dtype == np.float16:
data = data.astype(np.float32)

# TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32
if ftype == 1 and data_dtype == np.float16 and n_dims == 1:
data = data.astype(np.float32)

# if f16 desired, convert any float32 2-dim weight tensors to float16
if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2:
data = data.astype(np.float16)

print( name + ", shape " + str(len(data.shape)) + ", " + str(old_dtype) + " --> " + str(data.dtype))

gguf_writer.write_tensor_to_file(data)
print("gguf: write tensors")
gguf_writer.write_tensors_to_file()

gguf_writer.close()


print("gguf: model successfully exported to '" + fname_out + "'" )
print("gguf: model successfully exported to '" + fname_out + "'")
print("")
Loading