File size: 3,489 Bytes
4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9c2604e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 |
---
base_model:
- bamec66557/VICIOUS_MESH-12B-BETA
- bamec66557/VICIOUS_MESH-12B-OMEGA
library_name: transformers
tags:
- mergekit
- merge
---
# merge
This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
## Merge Details
### Merge Method
This model was merged using the SLERP merge method.
### Models Merged
The following models were included in the merge:
* [bamec66557/VICIOUS_MESH-12B-BETA](https://huggingface.co/bamec66557/VICIOUS_MESH-12B-BETA)
* [bamec66557/VICIOUS_MESH-12B-OMEGA](https://huggingface.co/bamec66557/VICIOUS_MESH-12B-OMEGA)
### Configuration
The following YAML configuration was used to produce this model:
```yaml
base_model: bamec66557/VICIOUS_MESH-12B-OMEGA
dtype: bfloat16
merge_method: slerp
tokenizer_source: base
# Slices Configuration
slices:
- sources:
- model: bamec66557/VICIOUS_MESH-12B-OMEGA
layer_range: [0, 10]
- model: bamec66557/VICIOUS_MESH-12B-BETA
layer_range: [0, 10]
parameters:
t:
- name: self_attn
value: [0.5, 0.55, 0.6, 0.65, 0.7]
- name: mlp
value: [1.0, 1.05, 1.1, 1.15, 1.2]
- name: layer_norm
value: [0.9, 0.95, 1.0, 1.05, 1.1]
- sources:
- model: bamec66557/VICIOUS_MESH-12B-OMEGA
layer_range: [10, 20]
- model: bamec66557/VICIOUS_MESH-12B-BETA
layer_range: [10, 20]
parameters:
t:
- name: self_attn
value: [0.4, 0.45, 0.5, 0.55, 0.6]
- name: mlp
value: [1.1, 1.15, 1.2, 1.25, 1.3]
- name: layer_norm
value: [1.0, 1.05, 1.1, 1.15, 1.2]
- sources:
- model: bamec66557/VICIOUS_MESH-12B-OMEGA
layer_range: [20, 30]
- model: bamec66557/VICIOUS_MESH-12B-BETA
layer_range: [20, 30]
parameters:
t:
- name: self_attn
value: [0.6, 0.65, 0.7, 0.75, 0.8]
- name: mlp
value: [0.9, 0.95, 1.0, 1.05, 1.1]
- name: layer_norm
value: [0.85, 0.9, 0.95, 1.0, 1.05]
- sources:
- model: bamec66557/VICIOUS_MESH-12B-OMEGA
layer_range: [30, 40]
- model: bamec66557/VICIOUS_MESH-12B-BETA
layer_range: [30, 40]
parameters:
t:
- name: self_attn
value: [0.7, 0.75, 0.8, 0.85, 0.9]
- name: mlp
value: [0.8, 0.85, 0.9, 0.95, 1.0]
- name: layer_norm
value: [0.8, 0.85, 0.9, 0.95, 1.0]
# Regularization
regularization:
- method: gradient_penalty
scale: 0.05 # Increased influence for gradient control
- method: weight_clipping
clip_range: [-0.2, 0.2] # Broader clipping range for flexibility
- method: random_noise
scale: 0.01 # Stronger noise injection
- method: attention_dropout
scale: 0.1 # Higher dropout to reduce attention fixation
# Postprocessing
postprocessing:
- operation: entropy_regularization
scale: 0.05 # Stronger encouragement for diverse outputs
- operation: non_linear_scaling
parameters:
function: tanh
- operation: sharpening
intensity: 0.5 # Enhanced sharpening for precise outputs
- operation: gaussian_smoothing
sigma: 1.5 # Increased smoothing for stable outputs
- operation: normalize
- operation: dynamic_scaling
scale_range: [0.8, 1.2] # Expanded dynamic range for scaling
- operation: smoothing
parameters:
adaptive: true
range: [0.85, 1.15] # Wider adaptive smoothing range
kernel_size: 5
```
|