Quantized using nncf 2.13.0

README.md

@@ -7,23 +7,22 @@ tags:
 
 This is an INT4 quantized version of the `Phi-3-mini-128k-instruct` model. The Python packages used in creating this model are as follows:
 ```
-openvino==2024.3.0.dev20240528
-openvino-nightly==2024.3.0.dev20240528
-openvino-tokenizers==2024.3.0.0.dev20240528
-optimum==1.19.2
-optimum-intel==1.17.0.dev0+aefabf0
-nncf==2.11.0.dev0+90a7f0d5
-torch==2.3.0+cu121
-transformers==4.40.2
+openvino==2024.4.0
+optimum==1.23.3
+optimum-intel==1.20.1
+nncf==2.13.0
+torch==2.5.1
+transformers==4.46.1
 ```
 This quantized model is created using the following command:
 ```
-optimum-cli export openvino -m "microsoft/Phi-3-mini-128k-instruct" --task text-generation-with-past --weight-format int4 --group-size 128 --trust-remote-code ./Phi-3-mini-128k-instruct-ov-int4 
+optimum-cli export openvino -m "microsoft/Phi-3-mini-128k-instruct" --task text-generation-with-past --weight-format int4 --group-size 128 --ratio 0.8 --trust-remote-code ./Phi-3-mini-128k-instruct-ov-int4
 ```
 For more details, run the following command from your Python environment: `optimum-cli export openvino --help`
 
 INFO:nncf:Statistics of the bitwidth distribution:
 |   Num bits (N) | % all parameters (layers)   | % ratio-defining parameters (layers)   |
 |----------------|-----------------------------|----------------------------------------|
-|              8 | 25% (27 / 130)              | 21% (25 / 128)                         |
-|              4 | 75% (103 / 130)             | 79% (103 / 128)                        |
+|              8 | 24% (23 / 130)              | 20% (21 / 128)                         |
+|              4 | 76% (107 / 130)             | 80% (107 / 128)                        |
+

config.json

@@ -1,8 +1,10 @@
 {
+  "_attn_implementation_autoset": true,
   "_name_or_path": "microsoft/Phi-3-mini-128k-instruct",
   "architectures": [
     "Phi3ForCausalLM"
   ],
+  "attention_bias": false,
   "attention_dropout": 0.0,
   "auto_map": {
     "AutoConfig": "microsoft/Phi-3-mini-128k-instruct--configuration_phi3.Phi3Config",
@@ -26,71 +28,64 @@
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
     "long_factor": [
-      1.0299999713897705,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0799999237060547,
-      1.2299998998641968,
-      1.2299998998641968,
-      1.2999999523162842,
-      1.4499999284744263,
-      1.5999999046325684,
-      1.6499998569488525,
-      1.8999998569488525,
-      2.859999895095825,
-      3.68999981880188,
-      5.419999599456787,
-      5.489999771118164,
-      5.489999771118164,
-      9.09000015258789,
-      11.579999923706055,
-      15.65999984741211,
-      15.769999504089355,
-      15.789999961853027,
-      18.360000610351562,
-      21.989999771118164,
-      23.079999923706055,
-      30.009998321533203,
-      32.35000228881836,
-      32.590003967285156,
-      35.56000518798828,
-      39.95000457763672,
-      53.840003967285156,
-      56.20000457763672,
-      57.95000457763672,
-      59.29000473022461,
-      59.77000427246094,
-      59.920005798339844,
-      61.190006256103516,
-      61.96000671386719,
-      62.50000762939453,
-      63.3700065612793,
-      63.48000717163086,
-      63.48000717163086,
-      63.66000747680664,
-      63.850006103515625,
-      64.08000946044922,
-      64.760009765625,
-      64.80001068115234,
-      64.81001281738281,
-      64.81001281738281
+      1.0700000524520874,
+      1.1200000047683716,
+      1.149999976158142,
+      1.4199999570846558,
+      1.5699999332427979,
+      1.7999999523162842,
+      2.129999876022339,
+      2.129999876022339,
+      3.009999990463257,
+      5.910000324249268,
+      6.950000286102295,
+      9.070000648498535,
+      9.930000305175781,
+      10.710000038146973,
+      11.130000114440918,
+      14.609999656677246,
+      15.409998893737793,
+      19.809999465942383,
+      37.279998779296875,
+      38.279998779296875,
+      38.599998474121094,
+      40.12000274658203,
+      46.20000457763672,
+      50.940006256103516,
+      53.66000747680664,
+      54.9373893737793,
+      56.89738845825195,
+      57.28738784790039,
+      59.98738479614258,
+      60.86738586425781,
+      60.887386322021484,
+      61.71739196777344,
+      62.91739273071289,
+      62.957393646240234,
+      63.41739273071289,
+      63.8173942565918,
+      63.83739471435547,
+      63.897396087646484,
+      63.93739700317383,
+      64.06739807128906,
+      64.11434936523438,
+      64.12435150146484,
+      64.15435028076172,
+      64.19435119628906,
+      64.24435424804688,
+      64.57435607910156,
+      64.69000244140625,
+      64.76000213623047
     ],
     "short_factor": [
-      1.05,
-      1.05,
-      1.05,
       1.1,
       1.1,
-      1.1500000000000001,
-      1.2000000000000002,
-      1.2500000000000002,
+      1.1,
       1.3000000000000003,
       1.3500000000000003,
-      1.5000000000000004,
-      2.000000000000001,
-      2.000000000000001,
-      2.000000000000001,
-      2.000000000000001,
+      1.3500000000000003,
+      1.4000000000000004,
+      1.5500000000000005,
       2.000000000000001,
       2.000000000000001,
       2.000000000000001,
@@ -111,26 +106,34 @@
       2.0500000000000007,
       2.0500000000000007,
       2.0500000000000007,
+      2.0500000000000007,
+      2.0500000000000007,
+      2.0500000000000007,
       2.1000000000000005,
       2.1000000000000005,
-      2.1000000000000005,
-      2.1500000000000004,
       2.1500000000000004,
-      2.3499999999999996,
-      2.549999999999999,
-      2.5999999999999988,
-      2.5999999999999988,
+      2.25,
+      2.25,
+      2.25,
+      2.25,
+      2.25,
+      2.3999999999999995,
+      2.4499999999999993,
+      2.499999999999999,
+      2.6999999999999984,
+      2.6999999999999984,
       2.7499999999999982,
-      2.849999999999998,
-      2.849999999999998,
-      2.9499999999999975
+      2.799999999999998,
+      2.8999999999999977,
+      3.049999999999997
     ],
-    "type": "su"
+    "type": "longrope"
   },
   "rope_theta": 10000.0,
   "sliding_window": 262144,
   "tie_word_embeddings": false,
-  "transformers_version": "4.40.2",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.1",
   "use_cache": true,
   "vocab_size": 32064
 }

generation_config.json

@@ -7,5 +7,5 @@
     32007
   ],
   "pad_token_id": 32000,
-  "transformers_version": "4.40.2"
+  "transformers_version": "4.46.1"
 }

openvino_detokenizer.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e1f06b61953c9af7ee2a9b4ba718b2233dc3ec25c64074ba80cdd6a842d18801
-size 499969

openvino_detokenizer.xml

@@ -1,97 +0,0 @@
-<?xml version="1.0"?>
-<net name="detokenizer" version="11">
-	<layers>
-		<layer id="0" name="Parameter_223816" type="Parameter" version="opset1">
-			<data shape="?,?" element_type="i64" />
-			<output>
-				<port id="0" precision="I64" names="Parameter_223816">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="1" name="Constant_223796" type="Const" version="opset1">
-			<data element_type="u8" shape="499969" offset="0" size="499969" />
-			<output>
-				<port id="0" precision="U8">
-					<dim>499969</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="2" name="Convert_223826" type="Convert" version="opset1">
-			<data destination_type="i32" />
-			<input>
-				<port id="0" precision="I64">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</input>
-			<output>
-				<port id="1" precision="I32">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="3" name="SentencepieceDetokenizer_223817" type="SentencepieceDetokenizer" version="extension">
-			<input>
-				<port id="0" precision="U8">
-					<dim>499969</dim>
-				</port>
-				<port id="1" precision="I32">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</input>
-			<output>
-				<port id="2" precision="I32">
-					<dim>-1</dim>
-				</port>
-				<port id="3" precision="I32">
-					<dim>-1</dim>
-				</port>
-				<port id="4" precision="U8">
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="4" name="StringTensorPack_223818" type="StringTensorPack" version="extension">
-			<data mode="begins_ends" />
-			<input>
-				<port id="0" precision="I32">
-					<dim>-1</dim>
-				</port>
-				<port id="1" precision="I32">
-					<dim>-1</dim>
-				</port>
-				<port id="2" precision="U8">
-					<dim>-1</dim>
-				</port>
-			</input>
-			<output>
-				<port id="3" precision="STRING" names="string_output">
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="5" name="Result_223819" type="Result" version="opset1">
-			<input>
-				<port id="0" precision="STRING">
-					<dim>-1</dim>
-				</port>
-			</input>
-		</layer>
-	</layers>
-	<edges>
-		<edge from-layer="0" from-port="0" to-layer="2" to-port="0" />
-		<edge from-layer="1" from-port="0" to-layer="3" to-port="0" />
-		<edge from-layer="2" from-port="1" to-layer="3" to-port="1" />
-		<edge from-layer="3" from-port="2" to-layer="4" to-port="0" />
-		<edge from-layer="3" from-port="3" to-layer="4" to-port="1" />
-		<edge from-layer="3" from-port="4" to-layer="4" to-port="2" />
-		<edge from-layer="4" from-port="3" to-layer="5" to-port="0" />
-	</edges>
-	<rt_info>
-		<eos_token_id value="32000" />
-	</rt_info>
-</net>

openvino_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:feff83396fe9946ae1dbf0cbe9c45486165c4ef064195426f5c836f84e8d7be0
-size 2450248212
+oid sha256:9ce9a3d5a7f07a4cb34ddae2c2d81c2b6f3cbac5c5cd51e1ba376bcf8983195e
+size 2432074068

openvino_tokenizer.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:46097e0534935f1aec4cbac2c90e565ec51a8513fcd53b841231849403e5e122
-size 499981

openvino_tokenizer.xml

@@ -1,231 +0,0 @@
-<?xml version="1.0"?>
-<net name="tokenizer" version="11">
-	<layers>
-		<layer id="0" name="string_input" type="Parameter" version="opset1">
-			<data shape="?" element_type="string" />
-			<output>
-				<port id="0" precision="STRING" names="string_input">
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="1" name="Constant_223802" type="Const" version="opset1">
-			<data element_type="i32" shape="" offset="0" size="4" />
-			<output>
-				<port id="0" precision="I32" />
-			</output>
-		</layer>
-		<layer id="2" name="Constant_223796" type="Const" version="opset1">
-			<data element_type="u8" shape="499969" offset="4" size="499969" />
-			<output>
-				<port id="0" precision="U8">
-					<dim>499969</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="3" name="SentencepieceTokenizer_223798" type="SentencepieceTokenizer" version="extension">
-			<data nbest_size="0" alpha="0" add_bos="true" add_eos="false" reverse="false" />
-			<input>
-				<port id="0" precision="U8">
-					<dim>499969</dim>
-				</port>
-				<port id="1" precision="STRING">
-					<dim>-1</dim>
-				</port>
-			</input>
-			<output>
-				<port id="2" precision="I64">
-					<dim>-1</dim>
-					<dim>2</dim>
-				</port>
-				<port id="3" precision="I32">
-					<dim>-1</dim>
-				</port>
-				<port id="4" precision="I64">
-					<dim>2</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="4" name="Broadcast_223803" type="Broadcast" version="opset3">
-			<data mode="numpy" />
-			<input>
-				<port id="0" precision="I32" />
-				<port id="1" precision="I64">
-					<dim>2</dim>
-				</port>
-			</input>
-			<output>
-				<port id="2" precision="I32">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="5" name="Constant_223804" type="Const" version="opset1">
-			<data element_type="i32" shape="" offset="499973" size="4" />
-			<output>
-				<port id="0" precision="I32" />
-			</output>
-		</layer>
-		<layer id="6" name="ShapeOf_223805" type="ShapeOf" version="opset3">
-			<data output_type="i64" />
-			<input>
-				<port id="0" precision="I32">
-					<dim>-1</dim>
-				</port>
-			</input>
-			<output>
-				<port id="1" precision="I64">
-					<dim>1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="7" name="Broadcast_223806" type="Broadcast" version="opset3">
-			<data mode="numpy" />
-			<input>
-				<port id="0" precision="I32" />
-				<port id="1" precision="I64">
-					<dim>1</dim>
-				</port>
-			</input>
-			<output>
-				<port id="2" precision="I32">
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="8" name="ScatterNDUpdate_223810" type="ScatterNDUpdate" version="opset4">
-			<input>
-				<port id="0" precision="I32">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-				<port id="1" precision="I64">
-					<dim>-1</dim>
-					<dim>2</dim>
-				</port>
-				<port id="2" precision="I32">
-					<dim>-1</dim>
-				</port>
-			</input>
-			<output>
-				<port id="3" precision="I32">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="9" name="ScatterNDUpdate_223810" type="Convert" version="opset1">
-			<data destination_type="i64" />
-			<input>
-				<port id="0" precision="I32">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</input>
-			<output>
-				<port id="1" precision="I64" names="attention_mask">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="11" name="Constant_223799" type="Const" version="opset1">
-			<data element_type="i32" shape="" offset="499977" size="4" />
-			<output>
-				<port id="0" precision="I32" />
-			</output>
-		</layer>
-		<layer id="12" name="Broadcast_223800" type="Broadcast" version="opset3">
-			<data mode="numpy" />
-			<input>
-				<port id="0" precision="I32" />
-				<port id="1" precision="I64">
-					<dim>2</dim>
-				</port>
-			</input>
-			<output>
-				<port id="2" precision="I32">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="13" name="ScatterNDUpdate_223801" type="ScatterNDUpdate" version="opset4">
-			<input>
-				<port id="0" precision="I32">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-				<port id="1" precision="I64">
-					<dim>-1</dim>
-					<dim>2</dim>
-				</port>
-				<port id="2" precision="I32">
-					<dim>-1</dim>
-				</port>
-			</input>
-			<output>
-				<port id="3" precision="I32">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="14" name="ScatterNDUpdate_223801" type="Convert" version="opset1">
-			<data destination_type="i64" />
-			<input>
-				<port id="0" precision="I32">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</input>
-			<output>
-				<port id="1" precision="I64" names="input_ids">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</output>
-		</layer>
-		<layer id="15" name="Result_223811" type="Result" version="opset1">
-			<input>
-				<port id="0" precision="I64">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</input>
-		</layer>
-		<layer id="10" name="Result_223812" type="Result" version="opset1">
-			<input>
-				<port id="0" precision="I64">
-					<dim>-1</dim>
-					<dim>-1</dim>
-				</port>
-			</input>
-		</layer>
-	</layers>
-	<edges>
-		<edge from-layer="0" from-port="0" to-layer="3" to-port="1" />
-		<edge from-layer="1" from-port="0" to-layer="4" to-port="0" />
-		<edge from-layer="2" from-port="0" to-layer="3" to-port="0" />
-		<edge from-layer="3" from-port="4" to-layer="4" to-port="1" />
-		<edge from-layer="3" from-port="3" to-layer="6" to-port="0" />
-		<edge from-layer="3" from-port="2" to-layer="8" to-port="1" />
-		<edge from-layer="3" from-port="4" to-layer="12" to-port="1" />
-		<edge from-layer="3" from-port="2" to-layer="13" to-port="1" />
-		<edge from-layer="3" from-port="3" to-layer="13" to-port="2" />
-		<edge from-layer="4" from-port="2" to-layer="8" to-port="0" />
-		<edge from-layer="5" from-port="0" to-layer="7" to-port="0" />
-		<edge from-layer="6" from-port="1" to-layer="7" to-port="1" />
-		<edge from-layer="7" from-port="2" to-layer="8" to-port="2" />
-		<edge from-layer="8" from-port="3" to-layer="9" to-port="0" />
-		<edge from-layer="9" from-port="1" to-layer="10" to-port="0" />
-		<edge from-layer="11" from-port="0" to-layer="12" to-port="0" />
-		<edge from-layer="12" from-port="2" to-layer="13" to-port="0" />
-		<edge from-layer="13" from-port="3" to-layer="14" to-port="0" />
-		<edge from-layer="14" from-port="1" to-layer="15" to-port="0" />
-	</edges>
-	<rt_info>
-		<eos_token_id value="32000" />
-	</rt_info>
-</net>

tokenizer_config.json

@@ -1,6 +1,7 @@
 {
-  "add_bos_token": true,
+  "add_bos_token": false,
   "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -116,7 +117,7 @@
     }
   },
   "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,