gguf.py : write tensors in a single pass (#2644)

* gguf : single pass for writing tensors + refactoring writer

* gguf : single pass for writing tensors + refactoring writer

* gguf : single pass for writing tensors + refactoring writer

* gguf : style fixes in simple conversion script

* gguf : refactor gptneox conversion script

* gguf : rename h5 to hf (for HuggingFace)

* gguf : refactor pth to gguf conversion script

* gguf : rm file_type key and method

* gguf.py : fix vertical alignment

* gguf.py : indentation

---------

Co-authored-by: Georgi Gerganov <[email protected]>

Author: monatis

convert-gptneox-h5-to-gguf.py renamed to convert-gptneox-hf-to-gguf.py

@@ -13,6 +13,8 @@
 from transformers import AutoTokenizer
 
 # ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
+
+
 def bytes_to_unicode():
     """
     Returns list of utf-8 byte and a corresponding list of unicode strings.
@@ -34,6 +36,7 @@ def bytes_to_unicode():
     cs = [chr(n) for n in cs]
     return dict(zip(bs, cs))
 
+
 def count_model_parts(dir_model: str) -> int:
     num_parts = 0
     for filename in os.listdir(dir_model):
@@ -44,6 +47,7 @@ def count_model_parts(dir_model: str) -> int:
         print("gguf: found " + str(num_parts) + " model parts")
     return num_parts
 
+
 if len(sys.argv) < 3:
     print("Usage: convert-h5-to-ggml.py dir-model ftype\n")
     print("  ftype == 0 -> float32")
@@ -58,7 +62,7 @@ def count_model_parts(dir_model: str) -> int:
 # possible tensor data types
 #   ftype == 0 -> float32
 #   ftype == 1 -> float16
-#
+
 # map from ftype to string
 ftype_str = ["f32", "f16"]
 
@@ -67,6 +71,7 @@ def count_model_parts(dir_model: str) -> int:
     ftype = int(sys.argv[2])
     if ftype < 0 or ftype > 1:
         print("Invalid ftype: " + str(ftype))
+
         sys.exit(1)
 
 fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf"
@@ -77,30 +82,30 @@ def count_model_parts(dir_model: str) -> int:
     hparams = json.load(f)
 
 if hparams["architectures"][0] != "GPTNeoXForCausalLM":
-    print("Model architecture not supported: " + hparams["architectures"][0] )
+    print("Model architecture not supported: " + hparams["architectures"][0])
+
     sys.exit()
 
 # get number of model parts
 num_parts = count_model_parts(dir_model)
 
-gguf_writer = gguf.GGUFWriter.open(fname_out)
+llm_arch = "gptneox"
+gguf_writer = gguf.GGUFWriter(fname_out, arch=llm_arch)
 
 print("gguf: get model metadata")
 
-llm_arch    = "gptneox"
 block_count = hparams["num_hidden_layers"]
 
-gguf_writer.add_architecture(llm_arch)
+gguf_writer.add_architecture()
 gguf_writer.add_name(last_dir)
-gguf_writer.add_file_type( "All tensors F32" if ftype == 0 else "Most tensors F16, some F32")
-gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"])
-gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"])
-gguf_writer.add_block_count(llm_arch, block_count)
-gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"])
-gguf_writer.add_rope_dimension_count(llm_arch, int( hparams["rotary_pct"]*(hparams["hidden_size"]//hparams["num_attention_heads"])) )
-gguf_writer.add_head_count(llm_arch, hparams["num_attention_heads"])
-gguf_writer.add_parallel_residual(llm_arch, hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True)
-gguf_writer.add_layer_norm_eps(llm_arch, hparams["layer_norm_eps"])
+gguf_writer.add_context_length(hparams["max_position_embeddings"])
+gguf_writer.add_embedding_length(hparams["hidden_size"])
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_feed_forward_length(hparams["intermediate_size"])
+gguf_writer.add_rope_dimension_count(int(hparams["rotary_pct"]*(hparams["hidden_size"]//hparams["num_attention_heads"])))
+gguf_writer.add_head_count(hparams["num_attention_heads"])
+gguf_writer.add_parallel_residual(hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True)
+gguf_writer.add_layer_norm_eps(hparams["layer_norm_eps"])
 
 # TOKENIZATION
 
@@ -124,14 +129,14 @@ def count_model_parts(dir_model: str) -> int:
 
     print("gguf: get gpt2 tokenizer vocab")
 
-    vocab_size = len( tokenizer_json["model"]["vocab"] )
+    vocab_size = len(tokenizer_json["model"]["vocab"])
 
     # ref: https://github.com/cmp-nct/ggllm.cpp/blob/master/falcon_convert.py
     tokenizer = AutoTokenizer.from_pretrained(dir_model)
 
     reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
     byte_encoder = bytes_to_unicode()
-    byte_decoder = {v:k for k, v in byte_encoder.items()}
+    byte_decoder = {v: k for k, v in byte_encoder.items()}
 
     for i in range(vocab_size):
         if i in reverse_vocab:
@@ -146,8 +151,9 @@ def count_model_parts(dir_model: str) -> int:
                         text.extend(c.encode('utf-8'))
         else:
             print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-            padding_token = f"[PAD{i}]".encode("utf8")
-            text = bytearray(padding_token)
+            pad_token = f"[PAD{i}]".encode("utf8")
+            text = bytearray(pad_token)
+
         tokens.append(text)
 
     gguf_writer.add_token_list(tokens)
@@ -201,7 +207,7 @@ def count_model_parts(dir_model: str) -> int:
     )
 
 for part_name in part_names:
-    print("gguf: loading model part '"+ part_name + "'")
+    print("gguf: loading model part '" + part_name + "'")
     model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu")
 
     for name in model_part.keys():
@@ -223,11 +229,12 @@ def count_model_parts(dir_model: str) -> int:
         elif name.endswith(".bias") and name[:-5] in tensor_map:
             name = tensor_map[name[:-5]] + ".bias"
         else:
-            print( "Can not map tensor '" + name + "'" )
+            print("Can not map tensor '" + name + "'")
             sys.exit()
 
         n_dims = len(data.shape)
         data_dtype = data.dtype
+        old_dtype = data_dtype
 
         # if f32 desired, convert any float16 to float32
         if ftype == 0 and data.dtype == np.float16:
@@ -241,77 +248,21 @@ def count_model_parts(dir_model: str) -> int:
         if ftype == 1 and data.dtype == np.float32 and name.endswith(".weight") and n_dims == 2:
             data_dtype = np.float16
 
-        data_nbytes = data.size * 2 if data_dtype == np.float16 else data.size * 4
+        print(name + ", n_dims = " + str(n_dims) + ", " + str(old_dtype) + " --> " + str(data_dtype))
+
+        data = data.astype(data_dtype)
 
-        gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes)
+        gguf_writer.add_tensor(name, data)
 
 
 print("gguf: write header")
 gguf_writer.write_header_to_file()
 print("gguf: write metadata")
 gguf_writer.write_kv_data_to_file()
-print("gguf: write tensor metadata")
-gguf_writer.write_ti_data_to_file()
-
-# tensor data
-print("gguf: convert and write tensor data")
-
-if num_parts == 0:
-    part_names = ("pytorch_model.bin",)
-else:
-    part_names = (
-        f"pytorch_model-{n:05}-of-{num_parts:05}.bin" for n in range(1, num_parts + 1)
-    )
-
-for part_name in part_names:
-    print("gguf: loading model part '"+ part_name + "'")
-    model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu")
-
-    for name in model_part.keys():
-        data = model_part[name]
-
-        old_dtype = data.dtype
-
-        # we don't need these
-        if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"):
-            continue
-
-        # convert any unsupported data types to float32
-        if data.dtype != torch.float16 and data.dtype != torch.float32:
-            data = data.to(torch.float32)
-
-        data = data.squeeze().numpy()
-
-        # map tensor names
-        if name.endswith(".weight") and name[:-7] in tensor_map:
-            name = tensor_map[name[:-7]] + ".weight"
-        elif name.endswith(".bias") and name[:-5] in tensor_map:
-            name = tensor_map[name[:-5]] + ".bias"
-        else:
-            print( "Can not map tensor '" + name + "'" )
-            sys.exit()
-
-        n_dims = len(data.shape)
-        data_dtype = data.dtype
-
-        # if f32 desired, convert any float16 to float32
-        if ftype == 0 and data.dtype == np.float16:
-            data = data.astype(np.float32)
-
-        # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32
-        if ftype == 1 and data_dtype == np.float16 and n_dims == 1:
-            data = data.astype(np.float32)
-
-        # if f16 desired, convert any float32 2-dim weight tensors to float16
-        if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2:
-            data = data.astype(np.float16)
-
-        print( name + ", shape " + str(len(data.shape)) + ", " + str(old_dtype) + " --> " + str(data.dtype))
-
-        gguf_writer.write_tensor_to_file(data)
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
 
 gguf_writer.close()
 
-
-print("gguf: model successfully exported to '" + fname_out + "'" )
+print("gguf: model successfully exported to '" + fname_out + "'")
 print("")