From e45425e00ae9746959ec333fff5c2c3c56ac4891 Mon Sep 17 00:00:00 2001 From: Yuanming Chen <122955822+CYMCharming@users.noreply.github.com> Date: Mon, 20 Oct 2025 14:29:32 +0800 Subject: [PATCH 1/2] add JSQ --- example/JSQ/README.md | 7 + example/JSQ/bash.sh | 4 + example/JSQ/build/lib/smoothquant/__init__.py | 1 + .../JSQ/build/lib/smoothquant/fake_quant.py | 190 ++++++ .../JSQ/build/lib/smoothquant/layerwrapper.py | 69 ++ example/JSQ/build/lib/smoothquant/quantize.py | 195 ++++++ example/JSQ/build/lib/smoothquant/smooth.py | 155 +++++ example/JSQ/build/lib/smoothquant/utils.py | 276 ++++++++ .../examples/__pycache__/jsq.cpython-310.pyc | Bin 0 -> 14802 bytes example/JSQ/examples/bash2.sh | 124 ++++ example/JSQ/examples/jsq.py | 634 ++++++++++++++++++ example/JSQ/examples/llama-7b-paddle.log | 99 +++ example/JSQ/examples/log10.txt | 0 example/JSQ/examples/run_examples.sh | 38 ++ example/JSQ/examples/smoothwanda.py | 44 ++ example/JSQ/examples/wikitext2.py | 206 ++++++ example/JSQ/examples/wikitext2.sh | 56 ++ example/JSQ/setup.py | 6 + example/JSQ/smoothquant.egg-info/PKG-INFO | 3 + example/JSQ/smoothquant.egg-info/SOURCES.txt | 12 + .../smoothquant.egg-info/dependency_links.txt | 1 + .../JSQ/smoothquant.egg-info/top_level.txt | 1 + example/JSQ/smoothquant/__init__.py | 1 + example/JSQ/smoothquant/__init__.pyc | Bin 0 -> 136 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 142 bytes .../__pycache__/fake_quant.cpython-310.pyc | Bin 0 -> 5096 bytes .../__pycache__/layerwrapper.cpython-310.pyc | Bin 0 -> 1911 bytes .../__pycache__/quantize.cpython-310.pyc | Bin 0 -> 2212 bytes .../__pycache__/smooth.cpython-310.pyc | Bin 0 -> 3553 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 6493 bytes example/JSQ/smoothquant/fake_quant.py | 190 ++++++ example/JSQ/smoothquant/layerwrapper.py | 69 ++ example/JSQ/smoothquant/quantize.py | 195 ++++++ example/JSQ/smoothquant/smooth.py | 155 +++++ example/JSQ/smoothquant/utils.py | 276 ++++++++ 35 files changed, 3007 insertions(+) create mode 100644 example/JSQ/README.md create mode 100644 example/JSQ/bash.sh create mode 100644 example/JSQ/build/lib/smoothquant/__init__.py create mode 100644 example/JSQ/build/lib/smoothquant/fake_quant.py create mode 100644 example/JSQ/build/lib/smoothquant/layerwrapper.py create mode 100644 example/JSQ/build/lib/smoothquant/quantize.py create mode 100644 example/JSQ/build/lib/smoothquant/smooth.py create mode 100644 example/JSQ/build/lib/smoothquant/utils.py create mode 100644 example/JSQ/examples/__pycache__/jsq.cpython-310.pyc create mode 100644 example/JSQ/examples/bash2.sh create mode 100644 example/JSQ/examples/jsq.py create mode 100644 example/JSQ/examples/llama-7b-paddle.log create mode 100644 example/JSQ/examples/log10.txt create mode 100644 example/JSQ/examples/run_examples.sh create mode 100644 example/JSQ/examples/smoothwanda.py create mode 100644 example/JSQ/examples/wikitext2.py create mode 100644 example/JSQ/examples/wikitext2.sh create mode 100644 example/JSQ/setup.py create mode 100644 example/JSQ/smoothquant.egg-info/PKG-INFO create mode 100644 example/JSQ/smoothquant.egg-info/SOURCES.txt create mode 100644 example/JSQ/smoothquant.egg-info/dependency_links.txt create mode 100644 example/JSQ/smoothquant.egg-info/top_level.txt create mode 100644 example/JSQ/smoothquant/__init__.py create mode 100644 example/JSQ/smoothquant/__init__.pyc create mode 100644 example/JSQ/smoothquant/__pycache__/__init__.cpython-310.pyc create mode 100644 example/JSQ/smoothquant/__pycache__/fake_quant.cpython-310.pyc create mode 100644 example/JSQ/smoothquant/__pycache__/layerwrapper.cpython-310.pyc create mode 100644 example/JSQ/smoothquant/__pycache__/quantize.cpython-310.pyc create mode 100644 example/JSQ/smoothquant/__pycache__/smooth.cpython-310.pyc create mode 100644 example/JSQ/smoothquant/__pycache__/utils.cpython-310.pyc create mode 100644 example/JSQ/smoothquant/fake_quant.py create mode 100644 example/JSQ/smoothquant/layerwrapper.py create mode 100644 example/JSQ/smoothquant/quantize.py create mode 100644 example/JSQ/smoothquant/smooth.py create mode 100644 example/JSQ/smoothquant/utils.py diff --git a/example/JSQ/README.md b/example/JSQ/README.md new file mode 100644 index 000000000..17a6fd997 --- /dev/null +++ b/example/JSQ/README.md @@ -0,0 +1,7 @@ +## FIMA-Q: Post-Training Quantization for Vision Transformers by Fisher Information Matrix Approximation + +## 1. 简介 + +本示例介绍了一种剪枝量化联合压缩方法。 + +技术详情见论文 [Compressing Large Language Models by Joint Sparsification and Quantization](https://openreview.net/attachment?id=sCGRhnuMUJ&name=pdf) \ No newline at end of file diff --git a/example/JSQ/bash.sh b/example/JSQ/bash.sh new file mode 100644 index 000000000..e425d626e --- /dev/null +++ b/example/JSQ/bash.sh @@ -0,0 +1,4 @@ +cd .. +python setup.py install +cd examples/ +python smoothquant_opt_demo.py \ No newline at end of file diff --git a/example/JSQ/build/lib/smoothquant/__init__.py b/example/JSQ/build/lib/smoothquant/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/example/JSQ/build/lib/smoothquant/__init__.py @@ -0,0 +1 @@ + diff --git a/example/JSQ/build/lib/smoothquant/fake_quant.py b/example/JSQ/build/lib/smoothquant/fake_quant.py new file mode 100644 index 000000000..28945ff1e --- /dev/null +++ b/example/JSQ/build/lib/smoothquant/fake_quant.py @@ -0,0 +1,190 @@ +from functools import partial + +import paddle +from loguru import logger + +############################## 相关utils函数，如下 ############################## + +def _Tensor_max(self, *args, **kwargs): + if "other" in kwargs: + kwargs["y"] = kwargs.pop("other") + ret = paddle.maximum(self, *args, **kwargs) + elif len(args) == 1 and isinstance(args[0], paddle.Tensor): + ret = paddle.maximum(self, *args, **kwargs) + else: + if "dim" in kwargs: + kwargs["axis"] = kwargs.pop("dim") + + if "axis" in kwargs or len(args) >= 1: + ret = paddle.max(self, *args, **kwargs), paddle.argmax(self, *args, **kwargs) + else: + ret = paddle.max(self, *args, **kwargs) + + return ret + +setattr(paddle.Tensor, "_max", _Tensor_max) + +def _Tensor_view(self, *args, **kwargs): + if args: + if len(args)==1 and isinstance(args[0], (tuple, list, str)): + return paddle.view(self, args[0]) + else: + return paddle.view(self, list(args)) + elif kwargs: + return paddle.view(self, shape_or_dtype = list(kwargs.values())[0]) + +setattr(paddle.Tensor, 'view', _Tensor_view) +############################## 相关utils函数，如上 ############################## + + + +@paddle.no_grad() +def quantize_weight_per_channel_absmax(w, n_bits=8): + # 获取每个channel的最大绝对值作为scale + scales = w.abs().max(keepdim=True, axis=-1)[0] + + # 计算量化的最大值 + q_max = 2 ** (n_bits - 1) - 1 + + # 关键修复：确保类型匹配 + # 方法1：将q_max转换为与scales相同的float类型 + scales = paddle.clip(scales, min=1e-05) / float(q_max) + + # 量化权重：除以scale，四舍五入，再乘回scale + w = paddle.round(w / scales) * scales + + return w + + +@paddle.no_grad() +def quantize_weight_per_tensor_absmax(w, n_bits=8): + scales = w.abs()._max() + q_max = 2 ** (n_bits - 1) - 1 + scales.clip_(min=1e-05).divide_(y=paddle.to_tensor(q_max)) + w.divide_(y=paddle.to_tensor(scales)).round_().multiply_(y=paddle.to_tensor(scales)) + return w + + +@paddle.no_grad() +def quantize_activation_per_token_absmax(t, n_bits=8): + t_shape = tuple(t.shape) + t.view(-1, t_shape[-1]) + scales = ( + t.abs().max(keepdim=True, axis=-1), + t.abs().argmax(keepdim=True, axis=-1), + )[0] + q_max = 2 ** (n_bits - 1) - 1 + scales.clip_(min=1e-05).divide_(y=paddle.to_tensor(q_max)) + t.divide_(y=paddle.to_tensor(scales)).round_().multiply_(y=paddle.to_tensor(scales)) + return t + + +@paddle.no_grad() +def quantize_activation_per_tensor_absmax(t, n_bits=8): + t_shape = tuple(t.shape) + t.view(-1, t_shape[-1]) + scales = t.abs()._max() + q_max = 2 ** (n_bits - 1) - 1 + scales.clip_(min=1e-05).divide_(y=paddle.to_tensor(q_max)) + t.divide_(y=paddle.to_tensor(scales)).round_().multiply_(y=paddle.to_tensor(scales)) + return t + + +class W8A8Linear(paddle.nn.Layer): + def __init__( + self, + in_features, + out_features, + bias=True, + act_quant="per_token", + quantize_output=False, + nbits=6, + ): + super().__init__() + self.nbits = nbits + self.in_features = in_features + self.out_features = out_features + out_0 = paddle.randn( + shape=[self.out_features, self.in_features], dtype="float16" + ) + out_0.stop_gradient = not False + self.register_buffer(name="weight", tensor=out_0) + if bias: + out_1 = paddle.zeros(shape=(1, self.out_features), dtype="float16") + out_1.stop_gradient = not False + self.register_buffer(name="bias", tensor=out_1) + else: + self.register_buffer(name="bias", tensor=None) + if act_quant == "per_token": + self.act_quant_name = "per_token" + self.act_quant = partial( + quantize_activation_per_token_absmax, n_bits=self.nbits + ) + elif act_quant == "per_tensor": + self.act_quant_name = "per_tensor" + self.act_quant = partial( + quantize_activation_per_tensor_absmax, n_bits=self.nbits + ) + else: + raise ValueError(f"Invalid act_quant: {act_quant}") + if quantize_output: + self.output_quant_name = self.act_quant_name + self.output_quant = self.act_quant + else: + self.output_quant_name = "None" + self.output_quant = lambda x: x + + def to(self, *args, **kwargs): + """Not Support auto convert *.to, please judge whether it is Pytorch API and convert by yourself""" + super(W8A8Linear, self).to(*args, **kwargs) + """Not Support auto convert *.to, please judge whether it is Pytorch API and convert by yourself""" + self.weight = self.weight.to(*args, **kwargs) + if self.bias is not None: + """Not Support auto convert *.to, please judge whether it is Pytorch API and convert by yourself""" + self.bias = self.bias.to(*args, **kwargs) + return self + + @paddle.no_grad() + def forward(self, x): + q_x = self.act_quant(x) + # wufazidong + # y = torch.functional.F.linear(q_x, self.weight, self.bias) + y = paddle.nn.functional.linear(q_x, self.weight, self.bias) + q_y = self.output_quant(y) + return q_y + + @staticmethod + def from_float( + module, + weight_quant="per_channel", + act_quant="per_token", + quantize_output=False, + nbits=8, + ): + assert isinstance(module, paddle.nn.Linear) + # import pdb; pdb.set_trace() + new_module = W8A8Linear( + module.weight.shape[0], + module.weight.shape[0], + module.bias is not None, + act_quant=act_quant, + quantize_output=quantize_output, + nbits=nbits, + ) + if weight_quant == "per_channel": + new_module.weight = quantize_weight_per_channel_absmax( + module.weight, n_bits=nbits + ) + elif weight_quant == "per_tensor": + new_module.weight = quantize_weight_per_tensor_absmax( + module.weight, n_bits=nbits + ) + else: + raise ValueError(f"Invalid weight_quant: {weight_quant}") + new_module.weight_quant_name = weight_quant + if module.bias is not None: + new_module.bias = module.bias + return new_module + + def __repr__(self): + return f"W8A8Linear({self.in_features}, {self.out_features}, bias={self.bias is not None}, weight_quant={self.weight_quant_name}, act_quant={self.act_quant_name}, output_quant={self.output_quant_name})" \ No newline at end of file diff --git a/example/JSQ/build/lib/smoothquant/layerwrapper.py b/example/JSQ/build/lib/smoothquant/layerwrapper.py new file mode 100644 index 000000000..33e59bdcd --- /dev/null +++ b/example/JSQ/build/lib/smoothquant/layerwrapper.py @@ -0,0 +1,69 @@ +import copy + +import paddle +from smoothquant.fake_quant import W8A8Linear +from smoothquant.utils import clip_matrix + +############################## 相关utils函数，如下 ############################## + +def _Tensor_reshape(self, *args, **kwargs): + if args: + if len(args) == 1 and isinstance(args[0], (tuple, list)): + return paddle.reshape(self, args[0]) + else: + return paddle.reshape(self, list(args)) + elif kwargs: + assert "shape" in kwargs + return paddle.reshape(self, shape=kwargs["shape"]) + +setattr(paddle.Tensor, "reshape", _Tensor_reshape) +############################## 相关utils函数，如上 ############################## + + + +class WrappedGPT: + """ + This class wraps a GPT layer for specific operations. + """ + + def __init__(self, layer, layer_id=0, layer_name="none"): + self.layer = layer + self.dev = self.layer.weight.place + self.rows = tuple(layer.weight.data.shape)[1] + self.columns = tuple(layer.weight.data.shape)[0] + self.inp_sum = None + self.inp_num = 0 + self.scaler_row = paddle.zeros(shape=self.columns) + self.nsamples = 0 + self.layer_id = layer_id + self.layer_name = layer_name + + def add_batch(self, inp, out): + if len(tuple(inp.shape)) == 2: + inp = inp.unsqueeze(axis=0) + tmp = tuple(inp.shape)[0] + if isinstance(self.layer, paddle.nn.Linear) or isinstance( + self.layer, W8A8Linear + ): + if len(tuple(inp.shape)) == 3: + inp = inp.reshape((-1, tuple(inp.shape)[-1])) + inp = inp.t() + if self.inp_sum is None: + self.inp_sum = copy.deepcopy(inp.t()) + else: + self.inp_sum += copy.deepcopy(inp.t()) + self.inp_num += 1 + self.scaler_row *= self.nsamples / (self.nsamples + tmp) + self.nsamples += tmp + inp = inp.astype("float32") + # self.scaler_row += paddle.linalg.norm(x=inp, p=2, axis=1) ** 2 / self.nsamples + # self.scaler_row += torch.norm(inp, p=2, dim=1) ** 2 / self.nsamples + # 用 elementwise add（避免 linalg.norm 的潜在兼容问题） + # import pdb + # import pdb; pdb.set_trace() + self.scaler_row = paddle.add(self.scaler_row, paddle.sum(inp * inp, axis=1) / float(self.nsamples)) + # self.scaler_row = self.scaler_row + paddle.sum(inp * inp, axis=1) / float(self.nsamples) + + + + diff --git a/example/JSQ/build/lib/smoothquant/quantize.py b/example/JSQ/build/lib/smoothquant/quantize.py new file mode 100644 index 000000000..2e49b2268 --- /dev/null +++ b/example/JSQ/build/lib/smoothquant/quantize.py @@ -0,0 +1,195 @@ +import paddle +import paddlenlp +from smoothquant.fake_quant import W8A8Linear +# from smoothquant.modeling_chatglm import GLMBlock + + +@paddle.no_grad() +def quantize_model( + model, weight_quant="per_tensor", act_quant="per_tensor", quantize_bmm_input=True +): + for name, m in model.named_sublayers(include_self=True): + # wufazidong + # if isinstance(m, transformers.models.opt.modeling_opt.OPTDecoderLayer): + if isinstance(m, paddlenlp.transformers.opt.modeling_opt.OPTDecoderLayer): + m.fc1 = W8A8Linear.from_float( + m.fc1, weight_quant=weight_quant, act_quant=act_quant + ) + m.fc2 = W8A8Linear.from_float( + m.fc2, weight_quant=weight_quant, act_quant=act_quant + ) + # wufazidong + # elif isinstance(m, transformers.models.opt.modeling_opt.OPTAttention): + elif isinstance(m, paddlenlp.transformers.opt.modeling_opt.OPTAttention): + m.q_proj = W8A8Linear.from_float( + m.q_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.k_proj = W8A8Linear.from_float( + m.k_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.v_proj = W8A8Linear.from_float( + m.v_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.out_proj = W8A8Linear.from_float( + m.out_proj, weight_quant=weight_quant, act_quant=act_quant + ) + # wufazidong + # elif isinstance(m, transformers.models.llama.modeling_llama.LlamaAttention): + elif isinstance(m, paddlenlp.transformers.LlamaAttention): + m.q_proj = W8A8Linear.from_float( + m.q_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.k_proj = W8A8Linear.from_float( + m.k_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.v_proj = W8A8Linear.from_float( + m.v_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.o_proj = W8A8Linear.from_float( + m.o_proj, weight_quant=weight_quant, act_quant=act_quant + ) + # wufazidong + # elif isinstance(m, transformers.models.llama.modeling_llama.LlamaMLP): + # 注意：这里假设LlamaMLP在PaddleNLP的LlamaForCausalLM模型中有类似的定义， + # 实际使用时需要确认PaddleNLP中是否有对应的类或模块，并可能需要调整导入路径。 + elif isinstance(m, paddlenlp.transformers.LlamaForCausalLM.LlamaMLP): + m.gate_proj = W8A8Linear.from_float( + m.gate_proj, weight_quant=weight_quant, act_quant=act_quant + ) + m.down_proj = W8A8Linear.from_float( + m.down_proj, weight_quant=weight_quant, act_quant=act_quant + ) + m.up_proj = W8A8Linear.from_float( + m.up_proj, weight_quant=weight_quant, act_quant=act_quant + ) + return model + + +@paddle.no_grad() +def quantize_layer( + module, + nbits, + weight_quant="per_channel", + act_quant="per_token", + quantize_bmm_input=True, +): + for name, m in module.named_sublayers(include_self=True): + # wufazidong + # if isinstance(m, transformers.models.opt.modeling_opt.OPTDecoderLayer): + if isinstance(m, paddlenlp.transformers.llama.modeling.LlamaAttention): + m.q_proj = W8A8Linear.from_float( + m.q_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + nbits=nbits, + ) + m.k_proj = W8A8Linear.from_float( + m.k_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + nbits=nbits, + ) + m.v_proj = W8A8Linear.from_float( + m.v_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + nbits=nbits, + ) + m.o_proj = W8A8Linear.from_float( + m.o_proj, weight_quant=weight_quant, act_quant=act_quant, nbits=nbits + ) + # wufazidong + # elif isinstance(m, transformers.models.llama.modeling_llama.LlamaDecoderLayer): + elif isinstance(m, paddlenlp.transformers.llama.modeling.LlamaDecoderLayer): + m.mlp.gate_proj = W8A8Linear.from_float( + m.mlp.gate_proj, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + m.mlp.down_proj = W8A8Linear.from_float( + m.mlp.down_proj, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + m.mlp.up_proj = W8A8Linear.from_float( + m.mlp.up_proj, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + elif "FalconDecoderLayer" in m.__class__.__name__: + m.self_attention.query_key_value = W8A8Linear.from_float( + m.self_attention.query_key_value, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + nbits=nbits, + ) + m.self_attention.dense = W8A8Linear.from_float( + m.self_attention.dense, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + m.mlp.dense_h_to_4h = W8A8Linear.from_float( + m.mlp.dense_h_to_4h, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + m.mlp.dense_4h_to_h = W8A8Linear.from_float( + m.mlp.dense_4h_to_h, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + # elif isinstance(m, GLMBlock) or "GLMBlock" in m.__class__.__name__: + # m.self_attention.query_key_value = W8A8Linear.from_float( + # m.self_attention.query_key_value, + # weight_quant=weight_quant, + # act_quant=act_quant, + # quantize_output=quantize_bmm_input, + # nbits=nbits, + # ) + # m.self_attention.dense = W8A8Linear.from_float( + # m.self_attention.dense, + # weight_quant=weight_quant, + # act_quant=act_quant, + # nbits=nbits, + # ) + # m.mlp.dense_h_to_4h = W8A8Linear.from_float( + # m.mlp.dense_h_to_4h, + # weight_quant=weight_quant, + # act_quant=act_quant, + # nbits=nbits, + # ) + # m.mlp.dense_4h_to_h = W8A8Linear.from_float( + # m.mlp.dense_4h_to_h, + # weight_quant=weight_quant, + # act_quant=act_quant, + # nbits=nbits, + # ) + return module \ No newline at end of file diff --git a/example/JSQ/build/lib/smoothquant/smooth.py b/example/JSQ/build/lib/smoothquant/smooth.py new file mode 100644 index 000000000..1edf2ff28 --- /dev/null +++ b/example/JSQ/build/lib/smoothquant/smooth.py @@ -0,0 +1,155 @@ +import paddle +import paddlenlp +# from smoothquant.modeling_chatglm import RMSNorm + +############################## 相关utils函数，如下 ############################## + +def _Tensor_view(self, *args, **kwargs): + if args: + if len(args)==1 and isinstance(args[0], (tuple, list, str)): + return paddle.view(self, args[0]) + else: + return paddle.view(self, list(args)) + elif kwargs: + return paddle.view(self, shape_or_dtype = list(kwargs.values())[0]) + +setattr(paddle.Tensor, 'view', _Tensor_view) +############################## 相关utils函数，如上 ############################## + + + +@paddle.no_grad() +def smooth_ln_fcs(ln, fcs, act_scales, alpha=0.5): + if not isinstance(fcs, list): + fcs = [fcs] + assert ( + isinstance(ln, paddle.nn.LayerNorm) + # wufazidong + # or isinstance(ln, transformers.models.llama.modeling_llama.LlamaRMSNorm) + or isinstance(ln, paddlenlp.transformers.llama.modeling.LlamaRMSNorm) + # or isinstance(ln, RMSNorm) + or "RMSNorm" in ln.__class__.__name__ + ) + # for fc in fcs: + # assert isinstance(fc, paddle.nn.Linear) + # assert ln.weight.size == fc.in_features == act_scales.size + device, dtype = fcs[0].weight.place, fcs[0].weight.dtype + act_scales = act_scales.to(device=device, dtype=dtype) + # import pdb; pdb.set_trace() + weight_scales = paddle.concat( + x=[ + ( + fc.weight.transpose([1,0]).abs().max(keepdim=True, axis=0), + fc.weight.transpose([1,0]).abs().argmax(keepdim=True, axis=0), + )[0] + for fc in fcs + ], + axis=0, + ) + weight_scales = (weight_scales.max(axis=0), weight_scales.argmax(axis=0))[0].clip( + min=1e-05 + ) + + scales = ( + (act_scales.pow(y=alpha) / weight_scales.pow(y=1 - alpha)) + .clip(min=1e-05) + .to(device) + .to(dtype) + ) + ln.weight.divide_(y=paddle.to_tensor(scales)) + if hasattr(ln, "bias"): + ln.bias.divide_(y=paddle.to_tensor(scales)) + for fc in fcs: + fc.weight.transpose_([1,0]).multiply_(y=paddle.to_tensor(scales.view(1, -1))).transpose_([1,0]) + + +@paddle.no_grad() +def smooth_lm(model, scales, alpha=0.5): + for name, module in model.named_sublayers(include_self=True): + # wufazidong + # if isinstance(module, transformers.models.opt.modeling_opt.OPTDecoderLayer): + if isinstance(module, paddlenlp.transformers.OPTDecoderLayer): + attn_ln = module.self_attn_layer_norm + qkv = [ + module.self_attn.q_proj, + module.self_attn.k_proj, + module.self_attn.v_proj, + ] + qkv_input_scales = scales[name + ".self_attn.q_proj"] + smooth_ln_fcs(attn_ln, qkv, qkv_input_scales, alpha) + ffn_ln = module.final_layer_norm + fc1 = module.fc1 + fc1_input_scales = scales[name + ".fc1"] + smooth_ln_fcs(ffn_ln, fc1, fc1_input_scales, alpha) + # wufazidong + # elif isinstance( + # module, transformers.models.llama.modeling_llama.LlamaDecoderLayer + # ): + elif isinstance( + module, paddlenlp.transformers.llama.modeling_llama.LlamaDecoderLayer + ): + attn_ln = module.input_layernorm + qkv = [ + module.self_attn.q_proj, + module.self_attn.k_proj, + module.self_attn.v_proj, + ] + if "layer" in name: + qkv_input_scales = scales[name + ".self_attn.q_proj"] + else: + qkv_input_scales = scales["self_attn.q_proj"] + smooth_ln_fcs(attn_ln, qkv, qkv_input_scales, alpha) + ffn_ln = module.post_attention_layernorm + fc1 = [module.mlp.gate_proj, module.mlp.up_proj] + if "layer" in name: + fc1_input_scales = scales[name + ".mlp.up_proj"] + else: + fc1_input_scales = scales["mlp.up_proj"] + smooth_ln_fcs(ffn_ln, fc1, fc1_input_scales, alpha) + # wufazidong + # elif isinstance(module, transformers.models.bloom.modeling_bloom.BloomBlock): + elif isinstance(module, paddlenlp.transformers.bloom.modeling_bloom.BloomBlock): + attn_ln = module.input_layernorm + qkv = module.self_attention.query_key_value + qkv_input_scales = scales[name + ".self_attention.query_key_value"] + smooth_ln_fcs(attn_ln, qkv, qkv_input_scales, alpha) + ffn_ln = module.post_attention_layernorm + fc1 = module.mlp.dense_h_to_4h + fc1_input_scales = scales[name + ".mlp.dense_h_to_4h"] + smooth_ln_fcs(ffn_ln, fc1, fc1_input_scales, alpha) + + +@paddle.no_grad() +def smooth_layer(name, module, scales, alpha=0.5): + # wufazidong + # if isinstance(module, transformers.models.llama.modeling_llama.LlamaDecoderLayer): + if isinstance(module, paddlenlp.transformers.llama.modeling.LlamaDecoderLayer): + # 注意：这里假设PaddleNLP中有对应的LlamaDecoderLayer类，实际使用时需要确认PaddleNLP的版本和API。 + attn_ln = module.input_layernorm + qkv = [ + module.self_attn.q_proj, + module.self_attn.k_proj, + module.self_attn.v_proj, + ] + qkv_input_scales = scales[name + ".self_attn.q_proj"] + smooth_ln_fcs(attn_ln, qkv, qkv_input_scales, alpha) + ffn_ln = module.post_attention_layernorm + fc1 = [module.mlp.gate_proj, module.mlp.up_proj] + fc1_input_scales = scales[name + ".mlp.gate_proj"] + smooth_ln_fcs(ffn_ln, fc1, fc1_input_scales, alpha) + elif "FalconDecoderLayer" in module.__class__.__name__: + attn_ln = module.input_layernorm + qkv_fc1 = [module.self_attention.query_key_value, module.mlp.dense_h_to_4h] + qkv_input_scales = scales[name + ".self_attention.query_key_value"] + smooth_ln_fcs(attn_ln, qkv_fc1, qkv_input_scales, alpha) + elif "GLMBlock" in module.__class__.__name__: + attn_ln = module.input_layernorm + qkv = [module.self_attention.query_key_value] + qkv_input_scales = scales[name + ".self_attention.query_key_value"] + smooth_ln_fcs(attn_ln, qkv, qkv_input_scales, alpha) + ffn_ln = module.post_attention_layernorm + fc1 = [module.mlp.dense_h_to_4h] + fc1_input_scales = scales[name + ".mlp.dense_h_to_4h"] + smooth_ln_fcs(ffn_ln, fc1, fc1_input_scales, alpha) + else: + raise TypeError(f"未添加的Decoder类{module}") \ No newline at end of file diff --git a/example/JSQ/build/lib/smoothquant/utils.py b/example/JSQ/build/lib/smoothquant/utils.py new file mode 100644 index 000000000..53125e0a3 --- /dev/null +++ b/example/JSQ/build/lib/smoothquant/utils.py @@ -0,0 +1,276 @@ +import copy + +import paddle +from smoothquant.fake_quant import W8A8Linear + +############################## 相关utils函数，如下 ############################## + +def dim2perm(ndim, dim0, dim1): + perm = list(range(ndim)) + perm[dim0], perm[dim1] = perm[dim1], perm[dim0] + return perm + +def _Tensor_reshape(self, *args, **kwargs): + if args: + if len(args) == 1 and isinstance(args[0], (tuple, list)): + return paddle.reshape(self, args[0]) + else: + return paddle.reshape(self, list(args)) + elif kwargs: + assert "shape" in kwargs + return paddle.reshape(self, shape=kwargs["shape"]) + +setattr(paddle.Tensor, "reshape", _Tensor_reshape) +############################## 相关utils函数，如上 ############################## + + + +def clip_matrix(matrix, abs=True, clip_l=0, clip_h=0, channel=False): + if clip_l == 0 and clip_h == 0: + return matrix + if channel: + matrix_flatten = matrix + if abs: + matrix_flatten = paddle.abs(x=matrix) + max_threshold = None + min_threshold = None + if clip_h != 0: + max_threshold = paddle.quantile( + x=matrix_flatten[0].astype(dtype="float64"), q=1 - clip_h, axis=0 + ) + clipped_matrix = paddle.clip(x=matrix, min=-max_threshold, max=max_threshold) + return clipped_matrix + else: + num_elements = matrix.size + if abs: + matrix_flatten = paddle.abs(x=matrix).flatten() + else: + matrix_flatten = matrix.flatten() + max_threshold = None + min_threshold = None + if clip_l != 0: + low_index = int(clip_l * num_elements) + min_threshold, _ = paddle.topk(x=matrix_flatten, largest=False, k=low_index) + min_threshold = min_threshold[-1] + if clip_h != 0: + high_index = int(clip_h * num_elements) + max_threshold, _ = paddle.topk(x=matrix_flatten, largest=True, k=high_index) + max_threshold = max_threshold[-1] + if abs: + clipped_matrix = paddle.clip( + x=matrix, min=-max_threshold, max=max_threshold + ) + else: + clipped_matrix = paddle.clip(x=matrix, min=min_threshold, max=max_threshold) + return clipped_matrix + + +def find_layers(module, layers=[paddle.nn.Linear, W8A8Linear], name=""): + if type(module) in layers or "FalconLinear" in module.__class__.__name__: + return {name: module} + else: + pass + res = {} + for name1, child in module.named_children(): + res.update( + find_layers( + child, layers=layers, name=name + "." + name1 if name != "" else name1 + ) + ) + return res + + +def check_sparsity(model): + CHATGLM = False + Falcon = False + if hasattr(model, "transformer"): + if hasattr(model.transformer, "embedding"): + CHATGLM = True + elif hasattr(model.transformer, "word_embeddings"): + Falcon = True + if CHATGLM: + layers = model.transformer.encoder.layers + elif Falcon: + layers = model.transformer.h + else: + layers = model.llama.layers + count = 0 + total_params = 0 + for i in range(len(layers)): + layer = layers[i] + subset = find_layers(layer) + sub_count = 0 + sub_params = 0 + for name in subset: + W = subset[name].weight.data + count += (W == 0).sum().item() + total_params += W.size + sub_count += (W == 0).sum().item() + sub_params += W.size + print(f"layer {i} sparsity {float(sub_count) / sub_params:.6f}") + return float(count) / total_params + + +def prepare_calibration_input(model, dataloader, device): + CHATGLM = False + Falcon = False + if hasattr(model, "transformer"): + if hasattr(model.transformer, "embedding"): + CHATGLM = True + elif hasattr(model.transformer, "word_embeddings"): + Falcon = True + use_cache = model.config.use_cache + model.config.use_cache = False + if CHATGLM: + layers = model.transformer.encoder.layers + elif Falcon: + layers = model.transformer.h + else: + layers = model.llama.layers + # if "model.embed_tokens" in model.hf_device_map: + # device = model.hf_device_map["model.embed_tokens"] + device = model.llama.embed_tokens.weight.place + dtype = next(iter(model.parameters())).dtype + inps = paddle.zeros( + shape=(128, model.seqlen, model.config.hidden_size), dtype=dtype + ) + inps.stop_gradient = not False + cache = {"i": 0, "attention_mask": None, "position_ids": None} + + class Catcher(paddle.nn.Layer): + def __init__(self, module): + super().__init__() + self.module = module + + def forward(self, inp, *args, **kwargs): + if CHATGLM: + inps[cache["i"]] = inp.transpose(perm=dim2perm(inp.ndim, 0, 1))[0] + else: + inps[cache["i"]] = inp + cache["i"] += 1 + if CHATGLM: + cache["attention_mask"] = args[0] + else: + # cache["attention_mask"] = kwargs["attention_mask"] + cache["attention_mask"] = kwargs.get("attention_mask", None) + if CHATGLM: + cache["position_ids"] = args[1] + elif Falcon: + pass + else: + # cache["position_ids"] = kwargs["position_ids"] + cache["position_ids"] = kwargs.get("position_ids", None) + raise ValueError + + layers[0] = Catcher(layers[0]) + for batch in dataloader: + try: + if cache["i"] == 0: + model_inputs = model.prepare_inputs_for_generation(batch[0].to(device)) + if "position_ids" in model_inputs: + cache["position_ids"] = model_inputs["position_ids"] + else: + cache["position_ids"] = paddle.arange( + start=0, end=tuple(batch[0].shape)[1], dtype="int64" + ).unsqueeze(axis=0) + model(batch[0].to(device)) + except ValueError: + pass + layers[0] = layers[0].module + outs = paddle.zeros_like(x=inps) + attention_mask = cache["attention_mask"] + position_ids = cache["position_ids"] + model.config.use_cache = use_cache + return inps, outs, attention_mask, position_ids + + +def return_given_alpha(alpha, sort_res, W_metric, tmp_metric, sum_before): + thres_cumsum = sum_before * alpha + sort_mask = tmp_metric <= thres_cumsum.reshape((-1, 1)) + thres = paddle.take_along_axis( + arr=sort_res[0], + axis=1, + indices=sort_mask.sum(dim=1, keepdims=True) - 1, + broadcast=False, + ) + W_mask = W_metric <= thres + cur_sparsity = (W_mask == True).sum() / W_mask.size + return W_mask, cur_sparsity + + +def cal_mse_layer( + args, layer1, layer2, inps, attention_mask, position_ids, outs1=None, outs2=None +): + if outs1 is None: + layer1_outs = [] + for i in range(args.nsamples): + with paddle.no_grad(): + layer1_outs.append( + layer1( + inps[i].unsqueeze(axis=0), + attention_mask=attention_mask, + position_ids=position_ids, + )[0][0] + ) + else: + layer1_outs = outs1 + if outs2 is None: + layer2_outs = [] + for i in range(args.nsamples): + with paddle.no_grad(): + layer2_outs.append( + layer2( + inps[i].unsqueeze(axis=0), + attention_mask=attention_mask, + position_ids=position_ids, + )[0][0] + ) + else: + layer2_outs = outs2 + mse = 0.0 + print(tuple(layer1_outs[0].shape)) + for i in range(args.nsamples): + device = layer2_outs[i].place + mse += paddle.nn.functional.mse_loss( + input=layer1_outs[i].to(device), label=layer2_outs[i] + ).item() + return mse + + +def generate_ss(activation, weight): + # weight: (out_features, in_features) for Paddle + cout, cin = weight.shape + ss = paddle.zeros_like(weight) + + # 确保 activation 是 2D + if len(activation.shape) == 1: + activation = activation.unsqueeze(0) + for i in range(cout): + w = weight.clone() + w[i, :] = 0 + + # w: (cout, cin), 需要转置为 (cin, cout) + # activation: (batch, cin) @ w.T(cin, cout) = (batch, cout) + # w_t = w.transpose([1, 0]) # 显式转置 + out = paddle.matmul(activation, w) + + max_values = paddle.max(out, axis=0) + min_values = paddle.min(out, axis=0) + + ss[i, :] = max_values - min_values + + ss = paddle.where( + paddle.isinf(ss), + paddle.to_tensor(100.0, dtype=ss.dtype), + ss + ) + ss = ss.transpose([1, 0]) + return ss + + +def generate_ss2(activation, weight): + out = activation @ weight.t() + max_values, _ = paddle.max(x=out, axis=0), paddle.argmax(x=out, axis=0) + min_values, _ = paddle.min(x=out, axis=0), paddle.argmin(x=out, axis=0) + row_ss = (max_values - min_values).reshape((-1, 1)) + return row_ss \ No newline at end of file diff --git a/example/JSQ/examples/__pycache__/jsq.cpython-310.pyc b/example/JSQ/examples/__pycache__/jsq.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c1adb9ff0b893ac14be8ef401edbeb45fa66f840 GIT binary patch literal 14802 zcmbt*d5|2}d0$^M({t?1?CiZbH~>KiLlOr`kc?3K4yF0UU z?dx6~;~tsPLdG%)9Llc57j4hVv}43ql*I9U$Brv^Qc4_koKz}yRSK2-lTuPiDALI9 z_j-=qHALB!p6b`{KEC(8@BO~-y=hNOq%=HV`Hlaz_L(;{?T;DQ{a6S*i`P4+Ynsr6 z-q4)rt2??18;+sAremtFtvfbC#Uko8u{j=GpYRXMxi<7Oeue&G2NVTW|ZG<%r@toIpt3_=C5etgwRFm9bKeF z=BDW^hzXIssX2@80XJPgD01%NmWkYXF^SkiLaQ8h7erx8U(v;sn7*ky4~UYO5wq|W z-E{xRjwa^B{7r33ca9Fi3t~})9~1|~LDX_g91@4|eMmeYiufKEN5oNlKO!C$$Hhlb z6Z-VBR(j+vsk&6sLtD6Rr`qn^fIrh{S43G<{EFxLB_o`8ZqsjHX|KDjTGy4~G=nd; zh1+ikr@7Vwx zv#H7Fhow~8Ry~eZKE>AiiJe_ND8-k$)@o<-TxeA{Ma41U56x<4^F94s$qrLBuh#PX zN~`LIX07FirsvBrA>B@+f~a_>B1FS=?29kG@)Bd5e=hP)th#i%br$7Gkr%pySpLVZTnw^I0ovwQu zr#d&nR8-;P=*AQ&eH>0kPvK`xr2eO^XLMV)bj!B?Z&tVB(_e?*%6tI7A?J~`%BdPY zoU$0+PvZ4f-~_rn3UcXNy3p4Rx#;U4xv`-6hHv`TiU|@f`mugIKpw{L7??s8_Dz;z z7}NfkAFCq|;tUcSwIQ!CZzEM;(o5FTU3Np#Dd|quTdQ>3a$A-K2?VN}4Ywt8NDX8D zCPp7xjT#0#$wC};XkD$j*TVSKN@LUYNLkNqtb|rYu6kj7{TjcDvXL{#BAN0Px8p)fpJWx)@(dFL6n9HvLe)9-3`@V-PqHUiyu@ z6fklD4z`9`6BB&B=0$9i49!lvqqxCIG!ft2Y|2GuS9}wXCOb5d6vn|1_&V{5=haqQ zB}*2W@d)dUp~3Ri(3GyfpJP(d5-g+W8D#a&!m)KOB^%4lHnMs_KZuvUX>d?cpT-Yg z_{mY0k>OG_)Cp)7jI#|sORZh)x^}7^j<9}5Z}_LwVv%40#G7I1YrKa3SwIc zvSfb>txc=3&Gcu*3>IB7NbZW|C}x>wI0}ERK3`wxFYaLDQUYmisXO7jo?mXa8aIkH zuh>nX5IEm_7#b0eS6<3C7UshV)N?SI;ze%^b z>|pwh~bos?F(UF@ix7-?fn(`TDOR^v3cE|U^l=AwOWo&6le~8ld%bka!dahMkqEd!a zjv$zVW(c?3N&`Ig8Px4P3#R}P#Gl&tDVXq5@T626{Dy(8ZPu98tr*Y&gcrpXF+&D9 z^1A#uDyb&m3?J@-93k<6_e&s>TD;ijAU2F`y>8;Q@W${$(9{#cxE;eXPhg31v0>k% zfaxc9w7*e+FQpc(wPn6;^^(0*FWt+$iiKz4M>yTGAJe=iuAvNnuW?aYy$K(JM{xJM zL|;842;eS>L~kOPke_5seWIT2Wg#@QdX7Fn4^ffb0uFgzds~0wf4^m^-q_nCK^bJX ztp%-@6Uive<5AbNw-Q1AdF_or-Q%KcrX}4l4 zhRBKhP5Z6*E&Y~p%e-aXir-4C#Koj2+_ZaBt6Fbb^=FFnFcfOr3DleG6@uyhTvT&? zK9~xioQUb$YVJ0^;^$d{^m*faZ)Pw9-qr!ub3rngS1buw6cQ7~@#Ah|68%X~- zlmXX%!9XnH=iplbuC1R?HR-)W!9mnq3J#&h1!0MUw=LF%77ru-;ov~P?<2wCsAc4t zy>0alVEoMWXEo!${9MsD1BSCR*t-*o* z3Go1EHKJ$#G-!DS@guq*e0)eXP)TuP{osyTz3A2BySU|B_i>9M58RIz{=d1!@Sfhy zEpHnen*Z2#EXZwZqPS&2mOKu>UVlOyxlKHp3X+CFnX>VknCTq>CC=8L>_5egJvahv zd332ePYoIJvRH-KT!yAxYqyG2m`he@x4cNZq5|OxNgNi&J$D28M|XapRToDUw5f^t z?h9&jl8+(tm2Oh)Jj4R;O{o3gRJ0YH+M{U^$5NU;f!8BS{T$TaH|!oXhi$D6#rMF0OWdhaEhn+?Bq5~}4Mc*<(2_e`;kZ8+2o z>qsvcoPgj~B04udU_!HC<>4!|?TCuS%Vz+hf3;?FuHdX#} zerg-kfRurLxs8=7u(krM8){V08_Yi&S4u41y@yA^r;;I$p>MJTCrqqhw)`g^zoUmS zrA>zVsZv@VU|LM+nM%D2&B`(~MdA9`)uRpnQPy#i`Jg5cZpaD75W9m@tX8GzI&ojA zkWOl?CWPB6Lnm?4)poN9{-T-y15Bz?^~)Z}?_#@GtWajrU5aQL<^;bS)%0c5?oltt zmaHG9&T!8%j1TOXfo**H-n7+S*gK=U%KjM!;}))_>L5I$tpgj%t@ug;b}a*&2N({^ z!d$A78;1}5qr%T4fLYTqTb$2`LoqMh^|y^}a{)88W%cy!*fuoVEv&-rICs@PLV)i| zPgJ1VSc;}67Ft|MAQW*>v*OFzby-F$PF%$_!X%6+<>iWBU4tsGmuYC}NUG?iM8x=) z;FsS9C)8_lnE{u9?6t^t6&a@n8oyc}rr+%RHnMosu2RtLQ~IplJ+_yYqgKZY2F27& zfu~e64HPfM_jN+n5vzJ2*O{(IdD*aBh4Wi@)smct1!n(}95vd_422Jc2aIag8W6T3 z8P-3U8H+XxEbPJ{*KUDu-g=nnw4sYISQ8#PN!3`sMo0M^`Unh3$8}YSlW0Kug>Iv= zp$Kbs;@~5KPz$KOvHC+l$mi|_jO1w+ep+F9Nqsj=(}+b0NO+wXh(!4bB+1KkC^F@X zbjW&PqSY?1%8GCn0J)(#xxmd3yGnsUr>W-4bIeVAFTa%zL9)C6CyYUPf>o08(Mhks zOg5NUtk%|zaAIv`XibA25E_14l4-GAVZUq%XHmMVuoAhl4CM$WAW;Uf{4%K;CZS<% zY`Sh2COhdi+gFiGmA&THR@Zz<#qC`M97M&-*0!;oh(RDt~A<^7_??M$#%<) zvN{Rct=iI8I*xpbWvIrLm6Gt15m!1IO-PaHqg4}S`WWM0 zfCKY6)?+lMVS=g})F^6h9WS(C4)J7%SvTlV8boSLObIh^&gL@u6q?v9BEBcfJJ8!& zFob%cUJvy(`B}zM6xOb)4c9S1H~ZCc6KYtk>crtAI~%o{mvm%}GIMu=wjH7%|t5fZl@>v9L;`QE#lhUCOjee=GrxJ{b z>)Cq;?^#x)JXtiP8dQ&ztw*0Wje990WgrX%ioeYJR@{aHW$9UCHfJRx1+1XQ?`6#t zYBP{#+%t^SpBh#cA+-5{VcG8+ZytWnc+*bPP+NdhVm=se|MXxS<bYOgP+2z@)rMV%jlE|n3zb!-6)0X$)NN%Cio=SN>Zbt*Xum?J47Cs6%s8%` z0W<~-7{tI2L+FH`MTxF>!EoI}XPpz0ioDkwILJ6pb?-*}sm#8S| z02MJ2hr-4()AgAxvzLH%NVT!0^Nadt0V@dSkN03Tia<5&N2q;T3v7`f)Sy2C)ltuS z$u88|AW1vftJ=o8rTKF=zBp7*H`cFdQ5#UOR2xyKzp$g?&$Gse8jKmx0zF);ALt(> zW#K;rm}B48dw@P72{4t$7lzkf#-H^M!?LB6*f%cqY-NA5sQl~@Ed7z7ZS4qs{6ays zsQ@ZC77OelZJ|C7$6e@UgNbcD$YMi33JPXNiXBP;Kxd(P{?}k4>V06J*FHP3yRSiN zTQ@*^KMPe{l?ok4$ zy71-p{0z>5t~OwdFPF!-SiX%ko)LeI1LT2L_xOkQ^3<@Ar7Kn19)}P8tb!f(&cK0=1uct+ z_B=EWp~rw=h4C~b9+oPQtuXP0mLV*N3+?lIDYg`mUtucFf!YfJ(Ndr0fz!vNY)ha* zHKH1mM|m83{c2<>1my4ten3y)xR~%2rPdH&4d#J`d8gXV>kytz`lJS9fhD1 z177$R_Kg;08^?@Bg1eHT&%Y=8%t-=??&czPj(rbm2JNwZxHq|4LX~9 z-3P7#Bn)wuBS9Z{J;@aoq1BkeX<>Sx&anrf(Q1XU%QPp;Ys~n!=#XzCeUNrhWfxX> zOMVxF1gxN`A~OycX4o`=K#$(XTE4^~|7CcU&N@bt@;lj}3H=`n4jSw_PNg@ScjhDP_c=EC1v(0)DS0FQ7~25E)5SzA5-=YmJsIzZKZ;nCCp9+hCb&(^2M z8z);BLZrwaK_r|!8zKoCB|y@)F#<_tfTWol+ar)v9)YCUJs@dr2$E1|yl(dqrmxJ_ zM~8Co9{@l~4FFPl0FW{YAWiuPM*!(1^!gC?I9*-Tu|;lRx&Nxjqt?TKNniD8H->#4 zb_&Gc@Nqw2Qq%@u8d73v7hpO9z%&&BCRovNF=Ygq{sfRojb#9tcJ+Q2Wcn2~%A*4~ zb07-sn(?EEd2kp8bb@7!+%N!2!yc`TfYP8pJ&x`eU{eIdo@c0kGy+NBmIyBGhDb9X z29Z$u{xhvW6)e&Na14k9KH3A3W)wui3Bdp&;qY|?k!Eo+K)y!Z`ykRu)bj{lLL|&_ zUqPfOCm<4M9S{lOAw-%VKqUFsC^G&!mm=Oh4;G+$yod3*A1E3?AyZg4?OjkP);*w( z4F7^aNZv*>3iQbDq3?U?sLgY~ z1AT>Oz>5mc7z)pr5uO>`-Z9rLc^LTth=;gF-Rg-U{!zroRD3+Luqta^BC^))2Relx&c!3W0eJ7)UQdylhM^H)rBoqHO zou8tkkiisvcj){bIzLV4XXspm!{h3K&HB%ah~#w^Rse`orOb&D6tWclNdQW$xs~YD zMgBaq{9QV{ezgzl$S*VQ@6q}Dbp8Py5*icngY+w4eUZL7I552Po`PD_pXUS+YK%by z?JoNO#C%3S12vZ{Is_14S^Nk(@;ELZT7L);3;>0pJty^k2DW9^gHk+<7qBn{#=Iu7^Vklct>*&+ho6&7jkImV zOb!k&3x2i_{Te+gK-bPg>z;zXJvGqEal@ARpq6F(lR+VvTEV7>1F~-qvO#9WfC@Vq zi7QwH{pHZa{hKQ<;P7P>0#0IZKQmhPqqQHV&hy41j)pqVyodSx z8s*Hi);&UN`nB3Rbdu}-<735poE!Q_2d!msVP#Xc220NShCPbOWVoWXN z-&5UD_jc3)QUWGt<*{kArceH5WN}g|3rGcR&sKg+)8GygBjxAV@;_m1yq{HTMW%1L z#^^eo@210zEHqK+{<&25ct?}=C!nNv3VCtuF9Gooi^B|Ci19`ZUt3S%7%zj1fi{f0 zDHw~VAwX+tR|LD9E&=x1VOBBtGEOVPKbjL&>Be^NBgPKLbiFYMlIx*d&wlO)%7zX=}!c> zA0pCf38#WMOjR&T3LtMHY}Azw(!8uEfXM@6Jtkjo=^GcKvb$3t28MN{lo-e`bsz>( zL1rKZC|bDqvsnCtROA^hejAH_BFGLF|M}gc5J?IIWll_sNi~Y+_KX51)KL%OL2TbB zAe{D$;x{phFAYYqE9E|;$e~Zjv412=JpGtos80dY7Jy)LmvDs6g(nkG+G@ zvGmP4Fhc7~0K@OF#vh^cqjcV;qa^RxQHUhJe;A)&lf(dou<|SLKnjj=((kh(3P2}I zo+zt3iu{9s{KrgTK_!djAJ9kHHkP~cpE5|J)GopM&lsfLNQq!4zNQRErNVtB?>|RA zCjq7y3E8NLf64M=vH33;9LY}kuULwA@*#fJv7Q>%E^;VJ#3TCVIDXm{3UT9iwOX>cp#_3}A4_g?d9>--qZ0+W{AWg%e9v zR6BaeRa`Y?ylTEjyr~ES&MYD&Ql<9sjODG@I#)<1Rsx&w97(63?Ra@G&*0X}V^86N zJpS(kAe6@8cy%rMKgIAbBRF#H+~sw1+=rQBn?(3dnuhusmOo6PPWBU(8t<~a(yBoZ z!vXiu6JJ8G(vXTR6$i#x!*g`FVFQPgqVnI-IYMXOSdyczzsv#Ukp=`Yih8s!zp9GHG(c1s+$ zP{PEQl!TLii0b6OhjRse|0)9^onL_i)4m$aHymvA{{bhZ?omb8d(@a!sl5Z2e+d;X zbI1zggL4gavTJ`}#b?3c7&ddEZ| z{#JuCTEx!sy3Vdx@}s#FCP&~UG^$jNiAB`fD8-^P{7*3AGMzE}+BXm%saH9W6g<2` zto{mdPcpt_EG@~$>9OR0#7FHDYB=&gF}CDbxc9Rb8g0)pJ7PHkW0qfQ;tC4?$tF>; z<`hQI^ArUSw?u_Wofs}zAk*)#YQ?&o8fSK#9oydU+Wi2il4uV9=C&y}m0We^cU_~7 zF6l+rhorhj;!N*O9{vqZU9@ltyOScnx`yFQ?oQbBYYk8R$x0qY<6#2*S6A5|VYg4S zKhb5uGDf2=UCU>ga+MD4%kmfLjOIqI2gMzV#=BOOni(Zb?eo!D4^NOZ@1t;R%uVzg qs;^XF>5syB(=KF_wqc#fo2ioaUGWDYem{d-;}!c6`*}N;o&JB*XR7}I literal 0 HcmV?d00001 diff --git a/example/JSQ/examples/bash2.sh b/example/JSQ/examples/bash2.sh new file mode 100644 index 000000000..17033dce1 --- /dev/null +++ b/example/JSQ/examples/bash2.sh @@ -0,0 +1,124 @@ +save_path="./log10.txt" +python_script="smoothquant_opt_demo.py" + +if [ ! -e "$save_path" ]; then + echo "文件不存在，正在创建..." + touch "$save_path" +fi + +cd .. +python setup.py build develop + +cuda_device=2 +# set environment +export CUDA_VISIBLE_DEVICES=$cuda_device + +cd examples/ + +#clip_hs=(0.0001 0.001 0.005 0.01 0.02 0.03 0.0 0.05 0.1) +clip_hs=(0.05) +#rhos=(0 0.0001 0.0005 0.001 0.005 0.01 0.05 0.1 0.5 1) +# for rho in "${rhos[@]}"; do +# echo "Params: $rho" +# python smoothquant_opt_demo.py --rho "$rho" >> "$save_path" +# done +# for clip_h in "${clip_hs[@]}"; do +# echo "Params: $clip_h" +# python smoothquant_opt_demo.py --clip_h "$clip_h" --rho 0.001 +# done +nohup python smoothwanda.py /root/.paddlenlp/models/facebook/llama-7b wikitext2 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ-paddle/llama-7b-paddle-1' > llama-7b-paddle.log 2>&1 & +# python smoothwanda.py /root/.paddlenlp/models/facebook/llama-7b wikitext2 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ-paddle/llama-7b-paddle' +# python smoothquant_opt_demo.py +# nohup python smoothwanda.py /mnt/disk1/hg/huggingface/cache/models--decapoda-research--llama-7b-hf/snapshots/5f98eefcc80e437ef68d457ad7bf167c2c6a1348 wikitext2 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama-7b-new' > llama-7b.log 2>&1 & +# nohup python smoothwanda.py /mnt/disk1/hg/huggingface/cache/models--decapoda-research--llama-7b-hf/snapshots/5f98eefcc80e437ef68d457ad7bf167c2c6a1348 wikitext2 --sparsity_ratio 0.5 --sparsity_type '2:4' --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama-7b-24' > llama-7b-24.log 2>&1 & +# nohup python smoothwanda.py /mnt/disk1/hg/huggingface/cache/models--decapoda-research--llama-7b-hf/snapshots/5f98eefcc80e437ef68d457ad7bf167c2c6a1348 wikitext2 --sparsity_ratio 0.5 --sparsity_type '4:8' --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama-7b-48' > llama-7b-48.log 2>&1 & + +# nohup python smoothwanda.py /mnt/disk1/yg/llama/llama-2-7b-hf wikitext2 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama-2-7b-new' > llama2-7b.log 2>&1 & +# nohup python smoothwanda.py /mnt/disk1/yg/llama/llama-2-7b-hf wikitext2 --sparsity_ratio 0.5 --sparsity_type '2:4' --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama2-7b-24' > llama2-7b-24.log 2>&1 & +# nohup python smoothwanda.py /mnt/disk1/yg/llama/llama-2-7b-hf wikitext2 --sparsity_ratio 0.5 --sparsity_type '4:8' --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama2-7b-48' > llama2-7b-48.log 2>&1 & + +# nohup python smoothwanda.py /mnt/disk1/wjy/Llama-2-13b-hf wikitext2 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama-2-13b-new' > llama2-13b.log 2>&1 & +# nohup python smoothwanda.py /mnt/disk1/hg/huggingface/cache/models--decapoda-research--llama-13b-hf/snapshots/438770a656712a5072229b62256521845d4de5ce wikitext2 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama-13b-new' > llama-13b.log 2>&1 & +# > llama-7b.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.4375 --rho 10 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho10' > llama-7b-rho10.log 2>&1 & +# nohup python smoothwanda.py yahma/llama-13b-hf c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-13b' > llama-13b.log 2>&1 & +# nohup python smoothwanda.py huggyllama/llama-30b c4 --test_only --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-30b' > llama-30b.log 2>&1 & +# python smoothwanda.py huggyllama/llama-30b c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-30b' +# nohup python smoothwanda.py meta-llama/Llama-2-7b-hf c4 --sparsity_ratio 0.5 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama2-7b-0-5' > llama2-7b-0-5.log 2>&1 & +# nohup python smoothwanda.py meta-llama/Llama-2-13b-hf c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama13-7b' > llama2-13b.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.55556 --rho 2.1 --nbits 6 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int6' > llama-7b-int6.log 2>&1 & +# nohup python smoothwanda.py THUDM/chatglm3-6b-base c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/chatglm-6b' > chatglm-6b.log 2>&1 & +# nohup python smoothwanda.py --test_only THUDM/chatglm3-6b-base c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/chatglm-6b' > chatglm-6b.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho20' > llama-7b-rho20.log 2>&1 & +# nohup python smoothwanda.py /mnt/nvme0/llm_weights/llama2/llama2-70b c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining2/JSQ/examples/llama2-70b' > llama2-70b.log 2>&1 & +# python smoothwanda.py /mnt/disk1/yg/llama/llama-2-7b-hf/ wikitext2 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 +# nohup python smoothwanda.py /mnt/nvme0/wangzining2/CodeLlama/CodeLlama-7b-hf c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining2/JSQ/examples/llama-7b-code' > llama-7b-code.log 2>&1 & + + +# python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --rho 2.1 +# python smoothwanda.py /mnt/nvme0/wangzining/smooth2/examples/save c4 --rho 2.1 +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.5 --sparsity_type '2:4' --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-2-4' > llama-7b-2-4.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.5 --sparsity_type '4:8' --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-4-8' > llama-7b-4-8.log 2>&1 & +# nohup python smoothwanda.py meta-llama/Llama-2-7b-hf c4 --sparsity_ratio 0.5 --sparsity_type '2:4' --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama2-7b-2-4' > llama2-7b-2-4.log 2>&1 & +# nohup python smoothwanda.py meta-llama/Llama-2-7b-hf c4 --sparsity_ratio 0.5 --sparsity_type '4:8' --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama2-7b-4-8' > llama2-7b-4-8.log 2>&1 & + + + +# tokenizer +# python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --rho 2.1 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b' +# python smoothwanda.py meta-llama/Llama-2-7b-chat-hf c4 --rho 2.1 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama2-7b' +# python smoothwanda.py yahma/llama-13b-hf c4 --rho 2.1 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-13b' +# python smoothwanda.py meta-llama/Llama-2-7b-hf c4 --rho 2.1 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama2-7b' +# python smoothwanda.py meta-llama/Llama-2-13b-hf c4 --rho 2.1 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama13-7b' + + +# int7 test +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.265625 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int7' > llama-7b-int7.log 2>&1 & + +# int5 test +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.609375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int5-1' > llama-7b-int5-1.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.305556 --rho 2.1 --nbits 6 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int5-2' > llama-7b-int5-2.log 2>&1 & + +# int3 test +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.75 --rho 2.1 --nbits 6 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int3-1' > llama-7b-int3-1.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.64 --rho 2.1 --nbits 5 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int3-2' > llama-7b-int3-2.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.4375 --rho 2.1 --nbits 4 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int3-3' > llama-7b-int3-3.log 2>&1 & + +# int2 test +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.88889 --rho 2.1 --nbits 6 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int2-1' > llama-7b-int2-1.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.84 --rho 2.1 --nbits 5 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int2-2' > llama-7b-int2-2.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.75 --rho 2.1 --nbits 4 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int2-3' > llama-7b-int2-3.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.55556 --rho 2.1 --nbits 3 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int2-4' > llama-7b-int2-4.log 2>&1 & +# nohup python smoothwanda.py huggyllama/llama-30b c4 --test_only --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-30b' > llama-30b.log 2>&1 & +# python smoothwanda.py huggyllama/llama-30b c4 --test_only --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-30b' + +# activation distribution +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --test_only --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-activation' > llama-7b-activation.log 2>&1 & +# python smoothwanda.py '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho10' c4 --test_only --sparsity_ratio 0.4375 --rho 10 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-activation-rho10' + +# gptq +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --gptq --test_only --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-gptq' > llama-7b-gptq.log 2>&1 & + +# ablation +# ours +# nohup python smoothwanda.py /mnt/disk1/hg/huggingface/cache/models--decapoda-research--llama-7b-hf/snapshots/5f98eefcc80e437ef68d457ad7bf167c2c6a1348 wikitext2 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama-7b-ablation' > llama-7b-ablation.log 2>&1 & +# w/o SAR +# nohup python smoothwanda.py /mnt/disk1/hg/huggingface/cache/models--decapoda-research--llama-7b-hf/snapshots/5f98eefcc80e437ef68d457ad7bf167c2c6a1348 wikitext2 --sparsity_ratio 0.4375 --rho 0 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama-7b' > llama-7b-norho.log 2>&1 & +# w/o clip +# nohup python smoothwanda.py /mnt/disk1/hg/huggingface/cache/models--decapoda-research--llama-7b-hf/snapshots/5f98eefcc80e437ef68d457ad7bf167c2c6a1348 wikitext2 --test_only --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama-7b-noclip' > llama-7b-noclip.log 2>&1 & +# w/o anneal +# nohup python smoothwanda.py /mnt/disk1/hg/huggingface/cache/models--decapoda-research--llama-7b-hf/snapshots/5f98eefcc80e437ef68d457ad7bf167c2c6a1348 wikitext2 --test_only --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/disk3/cym/JSQ/llama-7b-noanneal' > llama-7b-noanneal.log 2>&1 & + + +# rho +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.4375 --rho 1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho1' > llama-7b-rho1.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.4375 --rho 2 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho2' > llama-7b-rho2.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.4375 --rho 3 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho3' > llama-7b-rho3.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.4375 --rho 4 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho4' > llama-7b-rho4.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.4375 --rho 5 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho5' > llama-7b-rho5.log 2>&1 & + +# calibration +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --nsamples 32 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-cali32' > llama-7b-cali32.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --nsamples 64 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-cali64' > llama-7b-cali64.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --nsamples 16 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-cali16' > llama-7b-cali16.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --nsamples 8 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-cali8' > llama-7b-cali8.log 2>&1 & \ No newline at end of file diff --git a/example/JSQ/examples/jsq.py b/example/JSQ/examples/jsq.py new file mode 100644 index 000000000..5401bfff7 --- /dev/null +++ b/example/JSQ/examples/jsq.py @@ -0,0 +1,634 @@ +import math +import os +import pdb +import random +import time +from copy import deepcopy +from paddlenlp.datasets import load_dataset +import numpy as np +import paddle +import paddlenlp +from paddlenlp.transformers import AutoTokenizer, AutoModelForCausalLM +from loguru import logger +from smoothquant.layerwrapper import WrappedGPT +from smoothquant.quantize import quantize_layer +from smoothquant.smooth import smooth_layer +from smoothquant.utils import * + +# ############################# 相关utils函数，如下 ############################## +def device2str(type=None, index=None, *, device=None): + type = device if device else type + if isinstance(type, int): + type = f'gpu:{type}' + elif isinstance(type, str): + if 'cuda' in type: + type = type.replace('cuda', 'gpu') + if 'cpu' in type: + type = 'cpu' + elif index is not None: + type = f'{type}:{index}' + elif isinstance(type, paddle.CPUPlace) or (type is None): + type = 'cpu' + elif isinstance(type, paddle.CUDAPlace): + type = f'gpu:{type.get_device_id()}' + + return type + +def _Tensor_view(self, *args, **kwargs): + if args: + if len(args)==1 and isinstance(args[0], (tuple, list, str)): + return paddle.view(self, args[0]) + else: + return paddle.view(self, list(args)) + elif kwargs: + return paddle.view(self, shape_or_dtype = list(kwargs.values())[0]) + +setattr(paddle.Tensor, 'view', _Tensor_view) + +def paddle_max(*args, **kwargs): + if "input" in kwargs: + kwargs["x"] = kwargs.pop("input") + + out_v = None + if "out" in kwargs: + out_v = kwargs.pop("out") + + if "other" in kwargs: + kwargs["y"] = kwargs.pop("other") + ret = paddle.maximum(*args, **kwargs) + elif len(args)==2 and isinstance(args[1], paddle.Tensor): + ret = paddle.maximum(*args, **kwargs) + else: + if "dim" in kwargs: + kwargs["axis"] = kwargs.pop("dim") + + if "axis" in kwargs or len(args) >= 2: + if out_v: + ret = paddle.max(*args, **kwargs), paddle.argmax(*args, **kwargs) + paddle.assign(ret[0], out_v[0]) + paddle.assign(ret[1], out_v[1]) + return out_v + else: + ret = paddle.max(*args, **kwargs), paddle.argmax(*args, **kwargs) + return ret + else: + ret = paddle.max(*args, **kwargs) + return ret + + if out_v: + paddle.assign(ret, out_v) + return out_v + else: + return ret +############################## 相关utils函数，如上 ############################## + + + +global layer_num +layer_num = 0 + + +def prune_wanda_and_smoothquant_annealing(args, device=device2str("cuda:0")): + logger.info(f"test_only is {args.test_only}") + logger.info(f"rho is {args.rho}") + logger.info(f"sparsity_ratio is {args.sparsity_ratio}") + logger.info(f"nbits is {args.nbits}") + model = build_model(args) + CHATGLM = False + if CHATGLM: + logger.info("chatglm") + exit(0) + if hasattr(model, "transformer"): + if hasattr(model.transformer, "embedding"): + CHATGLM = True + logger.info("chatglm") + logger.info(f"sparsity type: {args.sparsity_type}") + prune_n, prune_m = 0, 0 + if args.sparsity_type != "unstructured": + assert ( + args.sparsity_ratio == 0.5 + ), "sparsity ratio must be 0.5 for structured N:M sparsity" + prune_n, prune_m = map(int, args.sparsity_type.split(":")) + logger.info(f"set n to {prune_n}, set m to {prune_m}") + # T = 300 + # Tmin = 10 + # k = 5 + # t = 0 + if CHATGLM: + layers = model.transformer.encoder.layers + else: + layers = model.llama.layers + clip_opts = [0.0, 4e-05, 5e-05, 6e-05, 7e-05] + clip_table = [2] * len(layers) + # ppl_prev = clip(model, args, clip_opts, clip_table, device, prune_n, prune_m) + clip(model, args, clip_opts, clip_table, device, prune_n, prune_m) + model.save_pretrained(args.saved_path) + tokenizer = AutoTokenizer.from_pretrained("facebook/llama-7b") + tokenizer.save_pretrained(args.saved_path) + logger.info(f"model saved to {args.saved_path}") + paddle.device.cuda.empty_cache() + + +def clip(model, args, clip_opts, clip_table, device, prune_n=0, prune_m=0): + logger.info("loading calibdation data") + dataloader, testenc = get_loaders( + args.dataset, + nsamples=args.nsamples, + seed=args.seed, + model=args.model, + seqlen=model.seqlen, + ) + logger.info("dataset loading complete") + with paddle.no_grad(): + inps, outs, attention_mask, position_ids = prepare_calibration_input( + model, dataloader, device + ) + CHATGLM = False + if hasattr(model, "transformer"): + if hasattr(model.transformer, "embedding"): + CHATGLM = True + if CHATGLM: + layers = model.transformer.encoder.layers + else: + layers = model.llama.layers + for i in range(len(layers)): + logger.info(f"layer is {i}") + layer = layers[i] + layer_name = f"model.layers.{i}" + subset = find_layers(layer) + # import pdb; pdb.set_trace() + if ( + any(s in args.model for s in ["30b", "70b"]) + and "model.layers.{i}" in model.hf_device_map + ): + logger.info("multi-dev") + dev = model.hf_device_map[f"model.layers.{i}"] + inps, outs, attention_mask, position_ids = ( + inps.to(dev), + outs.to(dev), + attention_mask.to(dev), + position_ids.to(dev), + ) + wrapped_layers = {} + for name in subset: + wrapped_layers[name] = WrappedGPT(subset[name]) + act_scales = {} + + def stat_tensor(name, tensor): + hidden_dim = tuple(tensor.shape)[-1] + tensor = tensor.view(-1, hidden_dim).abs().detach() + comming_max = ( + (paddle.max(x=tensor, axis=0), paddle.argmax(x=tensor, axis=0))[0] + .astype(dtype="float32") + .cpu() + ) + if name in act_scales: + act_scales[layer_name + "." + name] = paddle_max( + act_scales[name], comming_max + ) + else: + act_scales[layer_name + "." + name] = comming_max + + def add_batch(name): + def tmp(_, inp, out): + inp = inp[0].data + inp = clip_matrix(inp, args.abs, args.clip_l, clip_opts[clip_table[i]]) + stat_tensor(name, inp) + wrapped_layers[name].add_batch(inp, out.data) + + return tmp + + handles = [] + logger.info(f"nsamples is {args.nsamples}") + for name in wrapped_layers: + handles.append( + subset[name].register_forward_post_hook(hook=add_batch(name)) + ) + for j in range(args.nsamples): + with paddle.no_grad(): + if CHATGLM: + outs[j] = layer( + inps[j].unsqueeze(axis=0), attention_mask, position_ids + )[0] + else: + outs[j] = layer( + inps[j].unsqueeze(axis=0), + attention_mask=attention_mask, + position_ids=position_ids, + )[0] + for h in handles: + h.remove() + for name in subset: + weight = paddle.abs(subset[name].weight.data) + activation = paddle.sqrt(wrapped_layers[name].scaler_row.reshape((1, -1))) + ss = generate_ss( + wrapped_layers[name].inp_sum / wrapped_layers[name].inp_num, + subset[name].weight.data, + ) + # if weight.shape[0] != weight.shape[1]: + # import pdb; pdb.set_trace() + W_metric = weight.transpose([1,0]) * activation + args.rho * ss + + # 初始化为 float32，最后转换为 bool + W_mask = paddle.zeros_like(W_metric, dtype='float32') + + if prune_n != 0: + for ii in range(W_metric.shape[1]): + if ii % prune_m == 0: + tmp = W_metric[:, ii : ii + prune_m].astype(dtype="float32") + topk_indices = ii + paddle.topk(tmp, k=prune_n, axis=1, largest=False)[1] + # 使用 float32 进行 put_along_axis 操作 + W_mask = paddle.put_along_axis( + W_mask, + topk_indices, + paddle.ones_like(topk_indices, dtype='float32'), + axis=1 + ) + else: + # unstructured pruning + sorted_indices = paddle.argsort(W_metric, axis=-1, stable=True) + indices = sorted_indices[:, :int(W_metric.shape[1] * args.sparsity_ratio)] + # 使用 float32 进行 put_along_axis 操作 + W_mask = paddle.put_along_axis( + W_mask, + indices, + paddle.ones_like(indices, dtype='float32'), + axis=1 + ) + + # 转换为 bool 类型 + W_mask = W_mask.astype('bool') + + # 将 mask 位置的权重置零 + subset[name].weight.set_value( + paddle.where(W_mask.transpose([1,0]), paddle.zeros_like(subset[name].weight), subset[name].weight) + ) + + for j in range(args.nsamples): + with paddle.no_grad(): + if CHATGLM: + outs[j] = layer( + inps[j].unsqueeze(axis=0), attention_mask, position_ids + )[0] + else: + outs[j] = layer( + inps[j].unsqueeze(axis=0), + attention_mask=attention_mask, + position_ids=position_ids, + )[0] + smooth_layer(layer_name, layer, act_scales, 0.8) + quantize_layer(layer, nbits=args.nbits) + inps, outs = outs, inps + # logger.info("begin eval") + ppl = 0 + # if CHATGLM: + # ppl = chatglm_eval(model, testenc, device) + # else: + # ppl = llama_eval(model, testenc, device) + # logger.info(f"SmoothQuant W8A8 quantized model ppl: {ppl}") + # model.cpu() + paddle.device.cuda.empty_cache() + return ppl + + +@paddle.no_grad() +def chatglm_eval(model, testenc, dev): + model.eval() + testenc = testenc.input_ids + nsamples = testenc.size // model.seqlen + use_cache = model.config.use_cache + model.config.use_cache = False + layers = model.transformer.encoder.layers + model.transformer.embedding.word_embeddings = ( + model.transformer.embedding.word_embeddings.to(dev) + ) + model.transformer.rotary_pos_emb = model.transformer.rotary_pos_emb.to(dev) + layers[0] = layers[0].to(dev) + dtype = next(iter(model.parameters())).dtype + inps = paddle.zeros( + shape=(nsamples, model.seqlen, model.config.hidden_size), dtype=dtype + ) + cache = {"i": 0, "attention_mask": None} + + class Catcher(paddle.nn.Layer): + def __init__(self, module): + super().__init__() + self.module = module + + def forward( + self, + inp, + attention_mask=None, + position_ids=None, + kv_cache=None, + use_cache=True, + ): + inps[cache["i"]] = inp + cache["i"] += 1 + cache["attention_mask"] = attention_mask + cache["position_ids"] = position_ids + raise ValueError + + layers[0] = Catcher(layers[0]) + for i in range(nsamples): + batch = testenc[:, i * model.seqlen : (i + 1) * model.seqlen].to(dev) + try: + model(batch) + except ValueError: + pass + layers[0] = layers[0].module + layers[0] = layers[0].cpu() + model.transformer.embedding.word_embeddings = ( + model.transformer.embedding.word_embeddings.cpu() + ) + paddle.device.cuda.empty_cache() + outs = paddle.zeros_like(x=inps) + attention_mask = cache["attention_mask"] + position_ids = cache["position_ids"] + for i in range(len(layers)): + if ( + "30b" in model.config._name_or_path.lower() + and "model.layers.{i}" in model.hf_device_map + ): + dev = model.hf_device_map[f"model.layers.{i}"] + layer = layers[i].to(dev) + for j in range(nsamples): + outs[j] = layer(inps[j].unsqueeze(axis=0), attention_mask, position_ids)[0] + layers[i] = layer.cpu() + del layer + paddle.device.cuda.empty_cache() + inps, outs = outs, inps + if model.transformer.encoder.final_layernorm is not None: + model.transformer.encoder.final_layernorm = ( + model.transformer.encoder.final_layernorm.to(dev) + ) + model.transformer.output_layer = model.transformer.output_layer.to(dev) + testenc = testenc.to(dev) + nlls = [] + for i in range(nsamples): + hidden_states = inps[i].unsqueeze(axis=0) + if model.transformer.encoder.final_layernorm is not None: + hidden_states = model.transformer.encoder.final_layernorm(hidden_states) + lm_logits = model.transformer.output_layer(hidden_states) + shift_logits = lm_logits[:, :-1, :].contiguous() + shift_labels = testenc[:, i * model.seqlen : (i + 1) * model.seqlen][:, 1:] + loss_fct = paddle.nn.CrossEntropyLoss() + loss = loss_fct( + shift_logits.view(-1, shift_logits.shape[-1]), shift_labels.view(-1) + ) + neg_log_likelihood = loss.astype(dtype="float32") * model.seqlen + nlls.append(neg_log_likelihood) + ppl = paddle.exp(x=paddle.stack(x=nlls).sum() / (nsamples * model.seqlen)) + model.config.use_cache = use_cache + return ppl + + +@paddle.no_grad() +def llama_eval(model, testenc, dev): + model.eval() + testenc = testenc.input_ids + nsamples = testenc.size // model.seqlen + use_cache = model.config.use_cache + model.config.use_cache = False + layers = model.llama.layers + model.llama.embed_tokens = model.llama.embed_tokens.to(dev) #改前为 model.model.embed_tokens = model.model.embed_tokens.to(dev) + layers[0] = layers[0].to(dev) + dtype = next(iter(model.parameters())).dtype + inps = paddle.zeros( + shape=(nsamples, model.seqlen, model.config.hidden_size), dtype=dtype + ) + cache = {"i": 0, "attention_mask": None} + + class Catcher(paddle.nn.Layer): + def __init__(self, module): + super().__init__() + self.module = module + + # 改之前 + # def forward(self, inp, **kwargs): + # inps[cache["i"]] = inp + # cache["i"] += 1 + # cache["attention_mask"] = kwargs["attention_mask"] + # cache["position_ids"] = kwargs["position_ids"] + # raise ValueError + + def forward(self, *args, **kwargs): + inp = args[0] + inps[cache["i"]] = inp + cache["i"] += 1 + # Paddle 的 LlamaDecoderLayer 参数顺序为： + # hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, cache_position + if len(args) > 1: + cache["attention_mask"] = args[1] + if len(args) > 2: + cache["position_ids"] = args[2] + + layers[0] = Catcher(layers[0]) + for i in range(nsamples): + batch = paddle.to_tensor(testenc[:, i * model.seqlen : (i + 1) * model.seqlen], dtype='int64', place=dev) + #改之前 batch = testenc[:, i * model.seqlen : (i + 1) * model.seqlen].to(dev) + try: + model(batch) + except ValueError: + pass + layers[0] = layers[0].module + layers[0] = layers[0].cpu() + model.llama.embed_tokens = model.llama.embed_tokens.cpu() #改之前model.model.embed_tokens = model.model.embed_tokens.cpu() + paddle.device.cuda.empty_cache() + outs = paddle.zeros_like(x=inps) + attention_mask = cache["attention_mask"] + position_ids = cache["position_ids"] + for i in range(len(layers)): + if ( + any(s in model.config._name_or_path.lower() for s in ["30b", "70b"]) + and "model.layers.{i}" in model.hf_device_map + ): + logger.info("multi-dev") + dev = model.hf_device_map[f"model.layers.{i}"] + layer = layers[i].to(dev) + for j in range(nsamples): + outs[j] = layer( + inps[j].unsqueeze(axis=0), + attention_mask=attention_mask, + position_ids=position_ids, + )[0] + layers[i] = layer.cpu() + del layer + paddle.device.cuda.empty_cache() + inps, outs = outs, inps + if model.llama.norm is not None: + model.llama.norm = model.llama.norm.to(dev) + # 改之前if model.model.norm is not None: + # model.model.norm = model.model.norm.to(dev) + model.lm_head = model.lm_head.to(dev) + testenc = testenc.to(dev) + nlls = [] + for i in range(nsamples): + hidden_states = inps[i].unsqueeze(axis=0) + if model.model.norm is not None: + hidden_states = model.model.norm(hidden_states) + lm_logits = model.lm_head(hidden_states) + shift_logits = lm_logits[:, :-1, :].contiguous() + shift_labels = testenc[:, i * model.seqlen : (i + 1) * model.seqlen][:, 1:] + loss_fct = paddle.nn.CrossEntropyLoss() + loss = loss_fct( + shift_logits.view(-1, shift_logits.shape[-1]), shift_labels.view(-1) + ) + neg_log_likelihood = loss.astype(dtype="float32") * model.seqlen + nlls.append(neg_log_likelihood) + ppl = paddle.exp(x=paddle.stack(x=nlls).sum() / (nsamples * model.seqlen)) + model.config.use_cache = use_cache + return ppl + + +def get_wikitext2(nsamples, seed, seqlen, model): + # wufazidong + import datasets + # traindata = datasets.load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + # testdata = datasets.load_dataset("wikitext", "wikitext-2-raw-v1", split="test") + # traindata = load_dataset('wikitext', splits='train') + # testdata = load_dataset('wikitext', splits='test') + # traindata = load_dataset("glue", "sst-2", splits="train") + # testdata = load_dataset("glue", "sst-2", splits="test") + traindata, devdata, testdata = load_dataset("ptb") + if "glm" in model: + # wufazidong + # tokenizer = transformers.AutoTokenizer.from_pretrained( + # model, trust_remote_code=True, use_fast=False + # ) + tokenizer = paddlenlp.transformers.AutoTokenizer.from_pretrained( + model, trust_remote_code=True, use_fast=False + ) + else: + # tokenizer = transformers.LlamaTokenizer.from_pretrained(model, use_fast=False) + tokenizer = paddlenlp.transformers.LlamaTokenizer.from_pretrained(model, use_fast=False) + # trainenc = tokenizer("\n\n".join(traindata["sentence"]), return_tensors="pt") + train_text = "\n\n".join([example["sentence"] for example in traindata]) + trainenc = tokenizer(train_text, return_tensors="np") + # testenc = tokenizer("\n\n".join(testdata["sentence"]), return_tensors="pt") + test_text = "\n\n".join([example["sentence"] for example in testdata]) + testenc = tokenizer(test_text, return_tensors="np") + import random + + random.seed(seed) + trainloader = [] + for _ in range(nsamples): + i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1) + j = i + seqlen + inp = trainenc.input_ids[:, i:j] + # tar = inp.clone() + inp = paddle.to_tensor(inp) # 把 numpy 转成 Paddle Tensor + tar = inp.clone() + tar[:, :-1] = -100 + trainloader.append((inp, tar)) + return trainloader, testenc + + +def get_c4(nsamples, seed, seqlen, model): + logger.info("load c4 datasets") + logger.info("load from local") + # wufazidong + # traindata = datasets.load_from_disk( + # "/mnt/nvme0/wangzining/hf/allenai/c4/allenai--c4/train" + # ) + traindata = paddlenlp.datasets.load_from_disk( + "/mnt/nvme0/wangzining/hf/allenai/c4/allenai--c4/train" + ) + # wufazidong + # valdata = datasets.load_from_disk( + # "/mnt/nvme0/wangzining/hf/allenai/c4/allenai--c4/validation" + # ) + valdata = paddlenlp.datasets.load_from_disk( + "/mnt/nvme0/wangzining/hf/allenai/c4/allenai--c4/validation" + ) + if "glm" in model: + # wufazidong + # tokenizer = transformers.AutoTokenizer.from_pretrained( + # model, trust_remote_code=True, use_fast=False + # ) + tokenizer = paddlenlp.transformers.AutoTokenizer.from_pretrained( + model, use_auth_token=None # trust_remote_code 在 PaddleNLP 中没有直接对应参数，但可以通过设置 use_auth_token=None 来避免加载远程代码时的认证问题（如果不需要的话） + ) + else: + # wufazidong + # tokenizer = transformers.LlamaTokenizer.from_pretrained(model, use_fast=False) + tokenizer = paddlenlp.transformers.LlamaTokenizer.from_pretrained(model, use_fast=False) + import random + + random.seed(seed) + trainloader = [] + for _ in range(nsamples): + while True: + i = random.randint(0, len(traindata) - 1) + trainenc = tokenizer(traindata[i]["text"], return_tensors="pt") + if trainenc.input_ids.shape[1] >= seqlen: + break + i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1) + j = i + seqlen + inp = trainenc.input_ids[:, i:j] + tar = inp.clone() + tar[:, :-1] = -100 + trainloader.append((inp, tar)) + import random + + random.seed(0) + valenc = [] + for _ in range(256): + while True: + i = random.randint(0, len(valdata) - 1) + tmp = tokenizer(valdata[i]["text"], return_tensors="pt") + if tmp.input_ids.shape[1] >= seqlen: + break + i = random.randint(0, tmp.input_ids.shape[1] - seqlen - 1) + j = i + seqlen + valenc.append(tmp.input_ids[:, i:j]) + valenc = paddle.hstack(x=valenc) + + class TokenizerWrapper: + def __init__(self, input_ids): + self.input_ids = input_ids + + valenc = TokenizerWrapper(valenc) + return trainloader, valenc + + +def get_llama(model): + def skip(*args, **kwargs): + pass + + paddle.nn.initializer.KaimingUniform = skip + paddle.nn.initializer.Uniform = skip + paddle.nn.initializer.Normal = skip + kwargs = {"torch_dtype": "float16", "device_map": "auto"} + # wufazidong + # model = transformers.AutoModelForCausalLM.from_pretrained(model, **kwargs) + model = paddlenlp.transformers.AutoModelForCausalLM.from_pretrained(model, **kwargs) + model.seqlen = 2048 + return model + + +def build_model(args): + model_name = args.model + if "glm" in model_name: + kwargs = { + "torch_dtype": "float16", + "device_map": "auto", + "trust_remote_code": True, + } + else: + kwargs = { + # "torch_dtype": "float16", # caozuo1:注释掉，否则会报错。paddlenlp 不支持 torch_dtype（这是 HF Transformers 的参数 + # "device_map": "auto" # caozuo1:注释掉，否则会报错。paddlenlp 不支持 device_map（这是 HF Transformers 的参数 + } + # wufazidong + # model = transformers.AutoModelForCausalLM.from_pretrained(model_name, **kwargs) + model = paddlenlp.transformers.AutoModelForCausalLM.from_pretrained(model_name, **kwargs) + model.seqlen = args.seqlen + return model + + +def get_loaders(name, nsamples=128, seed=0, seqlen=2048, model=""): + if "wikitext2" in name: + return get_wikitext2(nsamples, seed, seqlen, model) + if "c4" in name: + return get_c4(nsamples, seed, seqlen, model) \ No newline at end of file diff --git a/example/JSQ/examples/llama-7b-paddle.log b/example/JSQ/examples/llama-7b-paddle.log new file mode 100644 index 000000000..645665281 --- /dev/null +++ b/example/JSQ/examples/llama-7b-paddle.log @@ -0,0 +1,99 @@ +which: no ccache in (/mnt/disk3/conda/envs/paddle_t/bin:/root/.local/bin:/root/.local/opt/make-4.3/bin:/root/.local/opt/bison-3.2.1/bin:/root/.local/opt/binutils-2.32/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/bin:/usr/local/cuda/bin:/root/.vscode-server/cli/servers/Stable-ddc367ed5c8936efe395cffeec279b04ffd7db78/server/bin/remote-cli:/root/.local/bin:/root/.local/opt/make-4.3/bin:/root/.local/opt/bison-3.2.1/bin:/root/.local/opt/binutils-2.32/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/bin:/usr/local/cuda/bin:/opt/rh/devtoolset-11/root/usr/bin:/root/.local/bin:/root/.local/opt/make-4.3/bin:/root/.local/opt/bison-3.2.1/bin:/root/.local/opt/binutils-2.32/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/bin:/root/.cargo/bin:/usr/local/cuda/bin:/home/anaconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/local/ssl/bin:/usr/lib64/mpich/bin:/usr/local/ssl/bin:/root/node-v18.20.3-linux-x64/bin:/usr/local/bin:/usr/lib64/mpich/bin:/root/bin:/root/.vscode-server/extensions/ms-python.debugpy-2025.14.1-linux-x64/bundled/scripts/noConfigScripts:/usr/local/ssl/bin:/usr/lib64/mpich/bin) +/mnt/disk3/conda/envs/paddle_t/lib/python3.10/site-packages/paddle/utils/cpp_extension/extension_utils.py:715: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md + warnings.warn(warning_message) +/mnt/disk3/conda/envs/paddle_t/lib/python3.10/site-packages/_distutils_hack/__init__.py:31: UserWarning: Setuptools is replacing distutils. Support for replacing an already imported distutils is deprecated. In the future, this condition will fail. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml + warnings.warn( +2025-10-20 13:26:53.930 | INFO | jsq:prune_wanda_and_smoothquant_annealing:92 - test_only is False +2025-10-20 13:26:53.930 | INFO | jsq:prune_wanda_and_smoothquant_annealing:93 - rho is 2.1 +2025-10-20 13:26:53.930 | INFO | jsq:prune_wanda_and_smoothquant_annealing:94 - sparsity_ratio is 0.4375 +2025-10-20 13:26:53.930 | INFO | jsq:prune_wanda_and_smoothquant_annealing:95 - nbits is 8 +[2025-10-20 13:26:53,930] [ INFO] - We are using to load '/root/.paddlenlp/models/facebook/llama-7b'. +[2025-10-20 13:26:53,931] [ INFO] - Loading configuration file /root/.paddlenlp/models/facebook/llama-7b/config.json +[2025-10-20 13:26:53,932] [ INFO] - Loading weights file /root/.paddlenlp/models/facebook/llama-7b/model.safetensors.index.json +W1020 13:26:53.938493 7813 gpu_resources.cc:114] Please NOTE: device: 0, GPU Compute Capability: 8.0, Driver API Version: 12.4, Runtime API Version: 11.8 + Loading checkpoint shards: 0%| | 0/5 [00:00 to load 'facebook/llama-7b'. +[2025-10-20 14:21:24,234] [ INFO] - tokenizer config file saved in /root/.paddlenlp/models/facebook/llama-7b/tokenizer_config.json +[2025-10-20 14:21:24,234] [ INFO] - Special tokens file saved in /root/.paddlenlp/models/facebook/llama-7b/special_tokens_map.json +[2025-10-20 14:21:24,234] [ INFO] - tokenizer config file saved in /mnt/disk3/cym/JSQ-paddle/llama-7b-paddle-1/tokenizer_config.json +[2025-10-20 14:21:24,234] [ INFO] - Special tokens file saved in /mnt/disk3/cym/JSQ-paddle/llama-7b-paddle-1/special_tokens_map.json +2025-10-20 14:21:24.235 | INFO | jsq:prune_wanda_and_smoothquant_annealing:128 - model saved to /mnt/disk3/cym/JSQ-paddle/llama-7b-paddle-1 diff --git a/example/JSQ/examples/log10.txt b/example/JSQ/examples/log10.txt new file mode 100644 index 000000000..e69de29bb diff --git a/example/JSQ/examples/run_examples.sh b/example/JSQ/examples/run_examples.sh new file mode 100644 index 000000000..54cd8774a --- /dev/null +++ b/example/JSQ/examples/run_examples.sh @@ -0,0 +1,38 @@ +# set environment +cuda_device=0 +export CUDA_VISIBLE_DEVICES=$cuda_device + +# nohup python examples.py \ +# --model huggyllama/llama-30b \ +# --prompt 'The extinction of the dinosaurs can be traced back a long time.' \ +# > prompts/llama-7b-dinner.log 2>&1 & + +# nohup python examples.py \ +# --model /mnt/nvme0/wangzining/smooth2/examples/llama-7b-int7 \ +# --prompt 'The universe is the entirety of space, time, matter, and energy that exists.' \ +# > prompts/llama-7b-dinner.log 2>&1 & + +# nohup python examples.py \ +# --model /mnt/nvme0/wangzining/smooth2/examples/llama-7b-int7 \ +# --prompt 'With the development of science and technology,' \ +# > prompts/llama-7b-dinner.log 2>&1 & + +nohup python examples.py \ + --model meta-llama/Llama-2-13b-hf \ + --prompt 'In 2008, Beijing hosted the Olympic Games.' \ + > prompts/llama-7b-dinner.log 2>&1 & + +# nohup python examples.py \ +# --model /mnt/nvme0/wangzining/smooth2/examples/llama2-7b \ +# --prompt 'The world is made of atoms.' \ +# > prompts/llama-7b-dinner.log 2>&1 & + +# nohup python examples.py \ +# --model THUDM/chatglm3-6b-base \ +# --prompt 'Write a poem about life.' \ +# > prompts/llama-7b-dinner.log 2>&1 & + +# nohup python examples.py \ +# --model /mnt/nvme0/wangzining/smooth2/examples/chatglm-6b \ +# --prompt 'Write a poem about life.' \ +# > prompts/llama-7b-dinner.log 2>&1 & \ No newline at end of file diff --git a/example/JSQ/examples/smoothwanda.py b/example/JSQ/examples/smoothwanda.py new file mode 100644 index 000000000..2c2d4dc64 --- /dev/null +++ b/example/JSQ/examples/smoothwanda.py @@ -0,0 +1,44 @@ +import argparse +from jsq import prune_wanda_and_smoothquant_annealing +# from smoothquant.prune_gptq import prepare_prune_wanda_and_gptq +parser = argparse.ArgumentParser() +parser.add_argument( + 'model', type=str, + help='LlaMa model to load; pass location of hugginface converted checkpoint.' + ) +parser.add_argument( + 'dataset', type=str, choices=['wikitext2', 'ptb', 'c4'], + help='Where to extract calibration data from.' +) +# parser.add_argument('--model', default="decapoda-research/llama-7b-hf", type=str, help='LLaMA model') +parser.add_argument('--seed', type=int, default=0, help='Seed for sampling the calibration data.') +parser.add_argument('--nsamples', type=int, default=1, help='Number of calibration samples.') +parser.add_argument('--seqlen', type=int, default=2048) +parser.add_argument('--sparsity_ratio', type=float, default=0.4375, help='Sparsity level') +parser.add_argument("--sparsity_type", default="unstructured", type=str, choices=["unstructured", "4:8", "2:4"]) +parser.add_argument("--prune_method", default="wanda", type=str, choices=["magnitude", "wanda", "sparsegpt"]) +parser.add_argument("--cache_dir", default="/mnt/disk1/hg/huggingface/cache", type=str) +parser.add_argument('--use_variant', action="store_true", + help="whether to use the wanda variant described in the appendix") +parser.add_argument('--save', type=str, default=None, help='Path to save results.') +parser.add_argument('--save_model', type=str, default=None, help='Path to save the pruned model.') +parser.add_argument('--clip_l', type=float, default=0.0) +parser.add_argument('--clip_h', type=float, default=0.001) +parser.add_argument('--abs', action="store_false") +parser.add_argument('--saved_path', type=str, default="./outputs/model_pruned") +parser.add_argument('--rho', type=float, default=0.0) +parser.add_argument('--nbits', type=int, default=8) +parser.add_argument('--test_only', action="store_true", default=False ) +parser.add_argument('--gptq', action="store_true", default=False) + +args = parser.parse_args() + +# annealing +if args.gptq: + # prepare_prune_wanda_and_gptq(args) + pass +else: + prune_wanda_and_smoothquant_annealing(args) + + + diff --git a/example/JSQ/examples/wikitext2.py b/example/JSQ/examples/wikitext2.py new file mode 100644 index 000000000..67b764d99 --- /dev/null +++ b/example/JSQ/examples/wikitext2.py @@ -0,0 +1,206 @@ +import argparse +from smoothquant.test import prune_wanda_and_smoothquant_annealing +from loguru import logger +import torch +import torch.nn as nn +from datasets import load_dataset +import random +from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + LlamaTokenizer, +) + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'model', type=str, + help='LlaMa model to load; pass location of hugginface converted checkpoint.' + ) +parser.add_argument( + 'dataset', type=str, choices=['wikitext2', 'ptb', 'c4'], + help='Where to extract calibration data from.' +) +# parser.add_argument('--model', default="decapoda-research/llama-7b-hf", type=str, help='LLaMA model') +parser.add_argument('--seed', type=int, default=0, help='Seed for sampling the calibration data.') +parser.add_argument('--nsamples', type=int, default=128, help='Number of calibration samples.') +parser.add_argument('--seqlen', type=int, default=2048) +parser.add_argument('--sparsity_ratio', type=float, default=0.4375, help='Sparsity level') +parser.add_argument("--sparsity_type", default="unstructured", type=str, choices=["unstructured", "4:8", "2:4"]) +parser.add_argument("--prune_method", default="wanda", type=str, choices=["magnitude", "wanda", "sparsegpt"]) +parser.add_argument("--cache_dir", default="/mnt/disk1/hg/huggingface/cache", type=str) +parser.add_argument('--use_variant', action="store_true", + help="whether to use the wanda variant described in the appendix") +parser.add_argument('--save', type=str, default=None, help='Path to save results.') +parser.add_argument('--save_model', type=str, default=None, help='Path to save the pruned model.') +parser.add_argument('--clip_l', type=float, default=0.0) +parser.add_argument('--clip_h', type=float, default=0.001) +parser.add_argument('--abs', action="store_false") +parser.add_argument('--saved_path', type=str) +parser.add_argument('--rho', type=float, default=0.0) +parser.add_argument('--nbits', type=int, default=8) +args = parser.parse_args() + +# annealing +# prune_wanda_and_smoothquant_annealing(args) + + +def build_model(args): + model_name = args.model + kwargs = {"torch_dtype": torch.float16, "device_map": "auto"} + while True: + try: + model = AutoModelForCausalLM.from_pretrained(model_name, **kwargs) + model.seqlen = args.seqlen + break + except Exception as e: + logger.info(f"Error: {e}") + return model + +def build_model_and_tokenizer(args, model_name): + while True: + try: + if "llama" in model_name: + tokenizer = LlamaTokenizer.from_pretrained(model_name, cache_dir=args.cache_dir) + else: + tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir=args.cache_dir) + break + except Exception as e: + print(f"Error: {e}") + kwargs = {"torch_dtype": torch.float16, "device_map": "auto"} + while True: + try: + model = AutoModelForCausalLM.from_pretrained(model_name, **kwargs, cache_dir=args.cache_dir) + model.seqlen = args.seqlen + break + except Exception as e: + print(f"Error: {e}") + return model, tokenizer + +def get_loaders( + name, nsamples=128, seed=0, seqlen=2048, model='' +): + tokenizer = LlamaTokenizer.from_pretrained(model, use_fast=False) + if 'wikitext2' in name: + return get_wikitext2(nsamples, seed, seqlen, model, tokenizer) + +def get_wikitext2(nsamples, seed, seqlen, model, tokenizer): + + # traindata = load_dataset('EleutherAI/wikitext_document_level', 'wikitext-2-raw-v1', split='train') + testdata = load_dataset('EleutherAI/wikitext_document_level', 'wikitext-2-raw-v1', split='test') + + # trainenc = tokenizer(" ".join(traindata['text']), return_tensors='pt') + testenc = tokenizer("\n\n".join(testdata['page']), return_tensors='pt') + + random.seed(seed) + + return None, testenc + +@torch.no_grad() +def llama_eval(model, testenc, dev=torch.device("cuda:0")): + + # logger.info('Evaluating ...') + model.eval() + model.seqlen = 280 + testenc = testenc.input_ids + nsamples = testenc.numel() // model.seqlen + + use_cache = model.config.use_cache + model.config.use_cache = False + layers = model.model.layers + + model.model.embed_tokens = model.model.embed_tokens.to(dev) + layers[0] = layers[0].to(dev) + + dtype = next(iter(model.parameters())).dtype + inps = torch.zeros( + (nsamples, model.seqlen, model.config.hidden_size), dtype=dtype, device=dev + ) + cache = {'i': 0, 'attention_mask': None} + + class Catcher(nn.Module): + def __init__(self, module): + super().__init__() + self.module = module + def forward(self, inp, **kwargs): + inps[cache['i']] = inp + cache['i'] += 1 + cache['attention_mask'] = kwargs['attention_mask'] + cache['position_ids'] = kwargs['position_ids'] + raise ValueError + layers[0] = Catcher(layers[0]) + for i in range(nsamples): + batch = testenc[:, (i * model.seqlen):((i + 1) * model.seqlen)].to(dev) + try: + model(batch) + except ValueError: + pass + layers[0] = layers[0].module + + layers[0] = layers[0].cpu() + model.model.embed_tokens = model.model.embed_tokens.cpu() + torch.cuda.empty_cache() + + outs = torch.zeros_like(inps) + attention_mask = cache['attention_mask'] + position_ids = cache['position_ids'] + + for i in range(len(layers)): + # print(i) + layer = layers[i].to(dev) + + # if args.nearest: + # subset = find_layers(layer) + # for name in subset: + # quantizer = Quantizer() + # quantizer.configure( + # args.wbits, perchannel=True, sym=False, mse=False + # ) + # W = subset[name].weight.data + # quantizer.find_params(W, weight=True) + # subset[name].weight.data = quantize( + # W, quantizer.scale, quantizer.zero, quantizer.maxq + # ).to(next(iter(layer.parameters())).dtype) + + for j in range(nsamples): + outs[j] = layer(inps[j].unsqueeze(0), attention_mask=attention_mask, position_ids=position_ids)[0] + layers[i] = layer.cpu() + del layer + torch.cuda.empty_cache() + inps, outs = outs, inps + + if model.model.norm is not None: + model.model.norm = model.model.norm.to(dev) + model.lm_head = model.lm_head.to(dev) + + testenc = testenc.to(dev) + nlls = [] + for i in range(nsamples): + # logger.info(f'{i}th sample') + hidden_states = inps[i].unsqueeze(0) + if model.model.norm is not None: + hidden_states = model.model.norm(hidden_states) + lm_logits = model.lm_head(hidden_states) + shift_logits = lm_logits[:, :-1, :].contiguous() + shift_labels = testenc[ + :, (i * model.seqlen):((i + 1) * model.seqlen) + ][:, 1:] + loss_fct = nn.CrossEntropyLoss() + loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + neg_log_likelihood = loss.float() * model.seqlen + nlls.append(neg_log_likelihood) + ppl = torch.exp(torch.stack(nlls).sum() / (nsamples * model.seqlen)) + # logger.info(ppl.item()) + + model.config.use_cache = use_cache + return ppl + + +model = build_model(args) + +dataloader, testenc = get_loaders( + args.dataset, nsamples=args.nsamples, seed=args.seed, model=args.model, seqlen=model.seqlen + ) +logger.info("dataset loading complete") +ppl = llama_eval(model, testenc) +logger.info(f'ppl is {ppl}') \ No newline at end of file diff --git a/example/JSQ/examples/wikitext2.sh b/example/JSQ/examples/wikitext2.sh new file mode 100644 index 000000000..38172018d --- /dev/null +++ b/example/JSQ/examples/wikitext2.sh @@ -0,0 +1,56 @@ +save_path="./log10.txt" +python_script="smoothquant_opt_demo.py" + +if [ ! -e "$save_path" ]; then + echo "文件不存在，正在创建..." + touch "$save_path" +fi + +cd .. +python setup.py build develop + +cuda_device=0 +# set environment +export CUDA_VISIBLE_DEVICES=$cuda_device + +cd examples/ + +#clip_hs=(0.0001 0.001 0.005 0.01 0.02 0.03 0.0 0.05 0.1) +clip_hs=(0.05) +#rhos=(0 0.0001 0.0005 0.001 0.005 0.01 0.05 0.1 0.5 1) +# for rho in "${rhos[@]}"; do +# echo "Params: $rho" +# python smoothquant_opt_demo.py --rho "$rho" >> "$save_path" +# done +# for clip_h in "${clip_hs[@]}"; do +# echo "Params: $clip_h" +# python smoothquant_opt_demo.py --clip_h "$clip_h" --rho 0.001 +# done + +# python smoothquant_opt_demo.py +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b' > llama-7b.log 2>&1 & +# nohup python smoothwanda.py yahma/llama-13b-hf c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-13b' > llama-13b.log 2>&1 & +# nohup python smoothwanda.py huggyllama/llama-30b c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-30b' > llama-30b.log 2>&1 & +# python smoothwanda.py huggyllama/llama-30b c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-30b' +# nohup python smoothwanda.py meta-llama/Llama-2-7b-hf c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama2-7b' > llama2-7b.log 2>&1 & +# nohup python smoothwanda.py meta-llama/Llama-2-13b-hf c4 --sparsity_ratio 0.4375 --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama13-7b' > llama2-13b.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.55556 --rho 2.1 --nbits 6 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-int6' > llama-7b-int6.log 2>&1 & + +# python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --rho 2.1 +# python smoothwanda.py /mnt/nvme0/wangzining/smooth2/examples/save c4 --rho 2.1 +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.5 --sparsity_type '2:4' --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-2-4' > llama-7b-2-4.log 2>&1 & +# nohup python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --sparsity_ratio 0.5 --sparsity_type '4:8' --rho 2.1 --nbits 8 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b-4-8' > llama-7b-4-8.log 2>&1 & + +# tokenizer +# python smoothwanda.py baffo32/decapoda-research-llama-7B-hf c4 --rho 2.1 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-7b' +# python smoothwanda.py meta-llama/Llama-2-7b-chat-hf c4 --rho 2.1 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama2-7b' +# python smoothwanda.py yahma/llama-13b-hf c4 --rho 2.1 --saved_path '/mnt/nvme0/wangzining/smooth2/examples/llama-13b' + +# python wikitext2.py /mnt/nvme0/wangzining/smooth2/examples/llama2-7b-2-4 wikitext2 +# python wikitext2.py /mnt/nvme0/wangzining/smooth2/examples/llama-13b wikitext2 + +python wikitext2.py /mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho1 wikitext2 +python wikitext2.py /mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho2 wikitext2 +python wikitext2.py /mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho3 wikitext2 +python wikitext2.py /mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho4 wikitext2 +python wikitext2.py /mnt/nvme0/wangzining/smooth2/examples/llama-7b-rho5 wikitext2 \ No newline at end of file diff --git a/example/JSQ/setup.py b/example/JSQ/setup.py new file mode 100644 index 000000000..c7a7c84a5 --- /dev/null +++ b/example/JSQ/setup.py @@ -0,0 +1,6 @@ +from struct import pack +from setuptools import setup, find_packages +setup( + name='smoothquant', + packages=find_packages(exclude=['figures', 'act_scales']) +) diff --git a/example/JSQ/smoothquant.egg-info/PKG-INFO b/example/JSQ/smoothquant.egg-info/PKG-INFO new file mode 100644 index 000000000..fea50873e --- /dev/null +++ b/example/JSQ/smoothquant.egg-info/PKG-INFO @@ -0,0 +1,3 @@ +Metadata-Version: 2.1 +Name: smoothquant +Version: 0.0.0 diff --git a/example/JSQ/smoothquant.egg-info/SOURCES.txt b/example/JSQ/smoothquant.egg-info/SOURCES.txt new file mode 100644 index 000000000..316574631 --- /dev/null +++ b/example/JSQ/smoothquant.egg-info/SOURCES.txt @@ -0,0 +1,12 @@ +README.md +setup.py +smoothquant/__init__.py +smoothquant/fake_quant.py +smoothquant/layerwrapper.py +smoothquant/quantize.py +smoothquant/smooth.py +smoothquant/utils.py +smoothquant.egg-info/PKG-INFO +smoothquant.egg-info/SOURCES.txt +smoothquant.egg-info/dependency_links.txt +smoothquant.egg-info/top_level.txt \ No newline at end of file diff --git a/example/JSQ/smoothquant.egg-info/dependency_links.txt b/example/JSQ/smoothquant.egg-info/dependency_links.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/example/JSQ/smoothquant.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/example/JSQ/smoothquant.egg-info/top_level.txt b/example/JSQ/smoothquant.egg-info/top_level.txt new file mode 100644 index 000000000..15a3563ec --- /dev/null +++ b/example/JSQ/smoothquant.egg-info/top_level.txt @@ -0,0 +1 @@ +smoothquant diff --git a/example/JSQ/smoothquant/__init__.py b/example/JSQ/smoothquant/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/example/JSQ/smoothquant/__init__.py @@ -0,0 +1 @@ + diff --git a/example/JSQ/smoothquant/__init__.pyc b/example/JSQ/smoothquant/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f8e7e8979ca739b58082712bdb37bcbba2f99a7 GIT binary patch literal 136 zcmZSn%**A*wmKu30SXv_v;zLsrtpa`S~Rog{6r=1@ZBjd6^~g@p=W7B^*FqHo5sJr8%i~AhU~s Gm;nGEJsb%D literal 0 HcmV?d00001 diff --git a/example/JSQ/smoothquant/__pycache__/__init__.cpython-310.pyc b/example/JSQ/smoothquant/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cffed8667f6ab3763517c80edb5d172e08405f04 GIT binary patch literal 142 zcmd1j<>g`k0Lsrtpa`S~Rog{6r=1@ZBjd6^~g@p=W7w>WHa^HWN5Qtd!S K6*B<|76t$o+8;sy literal 0 HcmV?d00001 diff --git a/example/JSQ/smoothquant/__pycache__/fake_quant.cpython-310.pyc b/example/JSQ/smoothquant/__pycache__/fake_quant.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..795b5a3bd309f6edb4ac42f1958fbfe1993c250d GIT binary patch literal 5096 zcmai2%X1t@8K3TX?r61oSWz4&6q}F)mN<4yNHE4WP9-GdQOd(E0t;a>-kz0ay*sn% z>9r%(>VjMqIEV|J2&r08TvWxCWB!r8CRLP=98whm#QeVQl`JWCn63V%=j-?Hd-j&I zv$2Ec$&Wu5kpYj&jF1f^e^K z`Hb?$On47ae}9XHB%rinKc{hdLOhLi95o4!|9Z2HNdsg^u$oFDjK5 zolFN;vZ5-bcF{uzsa(aoKN1||0P-Y7+fZh9?CO!d@oUC$(Tu>@;Gn5B9MBw`JAoO$(!J_C)T z&l_xsKgB%n_?I4+_tUXF>VHV;F4+50@8V<32?a?D4mon=DzEE8mM4@y zffOu%2TEqYt(@q4%;g#7lcd1olW@nuY?wp1+h`RI)Y4m#0ai0;Lm?a0S~|>(n|6qJ1?Q0(h^VS4z8;l9Fsyd)M>qh7PW${ZUr!x^jUAED||Jgs`<_88uzs z1B^yJ2u{@i@mQ11vw7~Z6U@6Un_$UrfZQTXI~-*1A?u*>JWBNz$k>@UJ5CQM2~OuC zUEk!;WF~^+&M|23ac9$2yaxox^tU-I!rfvIqL<nB)= z^crhbwAQja?=E|bE;Q_W_`Ek}zecOYY{GU3DWg{gDNaa@c@H=WZ=3a?XTZnZVZz^r ziyb>t4!4|10Ng{Ik9&`?VcAMJi+u!@{T0L}+=pi((7Z@#s zr&Q6THuivJZHlht5Z>TSV$3&V`b;@fbmJBI2=;bGehbv7668Qm^W=ft$tsSbwPVy8ro*{BX2#@XyA?>LAdaClW zFdD1MYgsXs!wVQ#kt2v9hj|qKC3n65&U;gtTtL&kW%52+|DQ}I9<Xau%Z7)NRL1t7F@lAw}c9=!A88fCX7R6p}R1=gFIqmoZc<1&EoWL<1zkT+( zv#;evmdY!-M!w5v!i%>k;B-Qey{Co>?aoE1{sW?%Er;9_phxgZAWTrWCQ#1?w4O)q zXqC*5Z}ZJixZ^NE^3jWgHx40aGE!HbL+=_A7?>(d()NweH=6bh$C2$@*s&q zJ&X4OgEd+;X+teZI{mb&lEm6Sc}Dp<2u11YJV;y%82vn$ zcW^s!8>lVed!99Tgf~ZtZ%6K4t>JkrqQ1PrHjdpz3Wuak21i1scu>&}748XRXA3#N znC;yum{X%T=pN)jSzYiv!A49zZrZ&g?J!Wd0lx9hN~xNcN5f$$RWlu_ve_w%>lr@2 zdrFljo5(5CDr@#eVl`{tSj*@V(9BgcubOYHt5SB>n$NxQaued-A9!`Wxn7PWCG;yT zU#?)|^2Z>WqyG!^Wd!j}su7Y0z%pf5;RD%Z{AyFE;@Q{2v z*rY_9E}~B;cv9fwA+_XVKkf;@LGz$SlD;qO+R{YKeKiS*}Pc|tSUD%jA**pltg^l{jI|0Zv zAM~*ozF@6ZAdmRdHSABmO{5LdV%8*%WSE>Dy-HCXeMP6ee;*DD1BOWcFL@2X$c0B$cv5}N+ literal 0 HcmV?d00001 diff --git a/example/JSQ/smoothquant/__pycache__/layerwrapper.cpython-310.pyc b/example/JSQ/smoothquant/__pycache__/layerwrapper.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a8a443d234877005216b96f9892d709f5dd782a7 GIT binary patch literal 1911 zcmZ8i&2Aev5GHrItJPYTCD}>sq(4gAi-6h&60}W#{-o$3Kw7vdY@mQHi|vw%R^Gpm zR2&nN!9M08`Xam)J^2B8>7nTi*LLMn;E*#z4rgb+8I`N6A%U^|`(M*P+JyXx z%<^EsJcg;B0pWzxjEtL=jw$d?=H%|!r6_YVmV09l_KbUP$k-PxL1zK?g};CkPY7@E z;0@uy!Wp+Y-6x~=1z06}BWl859)9=m$FvZ!1a2kC(kjYhEz?&C#WR<~h1G-k6HGM) zQd43?Z&{a z%F9wuU(91Tkj2MB9?7_>gxsl)&02II3RTMJ@)1ZTQD~6S4{49~97a2IK-oWX07BUT zlEAz%FuIRns*iyPf(afwoQ_@Iwkf4!#(my`)OtLCn1qlv>j8iA?BL|va0ugInyO)v z#Yzp)J2i}lpd4Cvhm%qcRV9*ik|x6vJdbr+7HY@dY(mS5vJeTv?rPv3J^LP}%7E0g zCK0L0jLhiFfxW9)O=0iNe8j;P=%UsQ8RQ;h0Y}>Acqh~=vkuq@xO{*qv0<1jkCbU8Wj4sC@P5CEteP{Q8C+1_k`6re`7d zt}S?5Wa6BIO^~mg6IjWt!^xb;AN1;3w~?M>0y_nylYw?w{v~lhW94hyx7?cLdSIt- zr~4kcOCI4lkiq?a?b>sFusz@{d|Ufx>u|OQ->?DTqZ=Y@JP7XkngpvLCcV#Fy5 z<2M~r+6SfJ@_+ zb8TFmSO14m8*W9v#yXjfz?kh01He%i`iW23yMen&0~!Li>2&EGkcad$|72qcRo4dg zMov>o2uA3ffKY^U6qzuJ@{-Rp+;^e~x?FatX+;q)lPHp(z#aJo5^QWs1i>!Xfo#H5 z7`OoXUdTEfD;!eN$Vt$XFqp?w9 zHQNH*B{p#+ZW8oJ={Np_jLKdgap#*mckc{mG4&XC=9J-##%aPvz9_zYOi*g@?!ucg zq$s%(z;cNC0^L)d{Q9kWs-GGe%8g77v_KDxzZXM=6DdSubmup0Sp~GyqL>p^pqePo)XtP9Hl}fEjr8ZRg;eV^LRjIO7snQ9q zK%L*pI^9ZCx5Tz9*mjBaDp>Ex%oW%&V{deZxI2sea6u?eX*lVt+Ut?m^nAkPmlHS>O6eo*pzR3DJycK&9B{*H|hu#TgDR*Y+EKV~o_9?gL9w8x( z!#Q^|=EdnGVUY;Zl6f|EqlD0K79U`+c2tvL+6$g9jq3cM}x`w83i7O58wA8<<| z4~7r9ec)xZ2(wA@G%nQIVlI`w$ql*uN_RS|?(9bqp2g50u8a0?6rCyTJ@`L=-yB47 zHXyU~@YcXTi3VRj`ucj&q(K@bNj8-^2Fpx0<|n+lG_WivHX?lnqN{f_9a+e_&@~G= z7ac=KI#T7&I@itD=_0oIA7b0J%PK`J>q32|YwQD91g{IK`%^EDX$bo*c%K~7*f+qr ze8iG1cqQw}ChO0D^~R}SuDQngU3rF0U_J0Q*K-8SH*&LNz8N?hxKqa0H}HCiyAnt8 z1~9QJxpi5w0qn}k$Y#*0#s*E?T6x+Vp7zSKwc!Ew5?F5l%sp1vFJ)Xe*p_jt{}Xg^ z!B#Wq2EDZ$UEEt`-&ylC@y^O~b;EPDz8MgiEZFUyg$p*Dzn+gfR8)( zz0gnMDhvB>FdJMfZAJCfOmcrhO@^BGRc7hM-2|o711?2{TuMOo5Ip8TA zLVSC=G;d3L+OKgF7Q6_l{JWK|z&BH2?+S5E2*K1syboH1fbxoa3|RUjh^{3FxTKki ztd}}ar6Ay|ET<%?S;ja((pTy3xeMf7dQ{{<-lg@rInFsy@*2AZ%YuE?V31-2#)rcA z2*gM=!l@d9r=AS6ym`~&<0{e*OIQ670fI)w}7S3SFXYmnczUH%7+b%}7&3x|b543F_=z$g) zz7g;vooRtKGLJRC>RZ13fE}@IYszu_lH%A?PT8+0j^ndUtvUA?RxNAlQi>^Rv^u6 z(2oOY#9^9AJmUs7O6$Oj2SM7@rI7}4TN<9|q|(~EOXKe}F%L^m*4Da7 zw&sWF-sjg^huyWCo8P)pXjx0Uy%vbw|vg&RHNeaKnh7A>I4a8uVS)J9n z!B&`YA{M|BbkmRU;`#emTXeINr-X@v_v?7m8~9?1Z^n*z>g24$h74=OH_{^myvu5Z z=tXwqL?z<*EZ3rPR5{Sn%Qv;R$2SsHqq*V^FspGsG=^r*qxsy(O^Wo8LucJLeDeY9 z-B5SnTi{!{6)hmU?1v1qMlox1x}oR=A6fK}L54?*x$c*h%%We(^(Bo~7pF2KSUP5) z<}n-E@UR-4%lPPgZXYv}WLkD1N2VfUjow2%`u-W6zlEAzo}*`z7B;MmG&X`js17O)l-dAn5yH7ddz@ zOq-^(?gn9JHw@1C@fpcrwEuYOkYc1xvdLF!N|%hxF9TfJ_7 z{nO+=F)fXom!4i?{J&i1Ppy;tpD1$HjT5)sN>{JNy_OfJ>zGW=!T`G`F5qyJsiE_) z-S>X_#}EFojsn!XVS<4jsxOhoPUxkNPw+G^3Bm_K!4s6pk|&8Q-SQ3tL9v&mqJEON zFRSYF+TaEvm1Z1!T~9jQo*%?v(vj6$#2DY1EV*tg_R`dKrQ^DZ*9}}(ERg}TAA7Ap zoKwcKp0rxMq~&GeBDrvhxJD}u`%>@s?n>Ja4?;h15!;@dsaloxu9tdQCd4O6xI7Iy zgt~(`3;Xe*tLn8`k~~hfbwo^-y;kO?2wDKwy|};YiA6FgXtThEmrrXv^Br&dRx&Te+R{BZFy>sh~uSs5&euDMLw>A?4(yyj)1_TT@$eQ7xKB z$^RT@>d&+1o4J0ViJe)yi2*RPp6S7Y@}LpD0G*9!vBQU@v4$h1!7hAK8XRbNzIW%M z^U(!(_t&X+OVRT9`%BOH{)zRKaRkcGj(`HC%ElI|s2V1%cz~p&AJLPART^9*WU{*N z_C+tcZ>+XkS5Y8TMEC3ew_RQB#{JbnzYwm>ES`LPsw_ydum_EUKpeVz!J(`Ee@`0f ztdBwn++7zn@rB*yMLAFP<-6%%N7+n^n*J0rND!pR`Pbjrx)HQcsX|q@T%0=PsuOOa zYJJL;R`DtL!74BD`&FY#RatL`i5JhFM2TG$O_SW|0Rewo-Htvxgy<9jKz%K0;~lyj%U zSdpgkU3`V&_GOBjL2;E9*aI{oHr+80SnB(G2eJ<07GE-k6|3E*ZE=&1))A3Y+h>8K zGy#(C%aBjEKo)sTw-^D0#f+y_fC$a1=ta0$CFsy6dk*#-PWyF$2+ixPfwK;ziCc{P zN5 zGli>4N_++f#5ajhXCq!A@+y(nhy z?y0n{BT*Z8DS;=m@;G^EY_J*^J2;AFra?nVbtgh$8a3q7PJ@iP0BIF1ytI>^+Yz2$ q1cTCEDVXt4`Ko#@=UP$I>qvhsrAENf9EPeYS*SGYCA?p%v;P65X_Q_7 literal 0 HcmV?d00001 diff --git a/example/JSQ/smoothquant/__pycache__/utils.cpython-310.pyc b/example/JSQ/smoothquant/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2d89bc538615430caa5efda3e0617fa741d6edd4 GIT binary patch literal 6493 zcmb7I-)|(zU9YOH?&;}ikG(#hcfGrJw}Dg09Emr(ce+bpbGqa%-XvTcqP19V2z1(0 zJ$8?KddAf~`@{6WaXz%WQY;jBi4;J#gpfAj0r7?+BwqO!>XoN^t{*5)8coGqckzEE?jjnX+V6^CZ?ihZ<);`T|PtA{DP8D_O}aw$JzZ)Tx|GtT5z!g$Y5=TT~O2-(gSrdPf+U zrA2969p_s~M;e){bZ8{9FWUTuk>f-UpE6B5`5&v-hk1EjrNz$c*Lx3#*S~e=yI&Z` zN@dCQVmKO=+q;t(0+TY$iq-K$<71$kG=NX*^N@x7GM8M3DE2tAlOKJQi}W3&VjEe- z^=C@t`&{uIp;z!(T;X#{Q96CGPc8D$tsp_IN8%2v>m@#OpHsaO`liC?-eX1QTkIoV z$^pp><*YCz4MF>W~-XNjgg zH(veerlFI%n`2~=WPKxJy;T@*=RQ5{&ed6qdd){0NnVU}H0w2o#$o}PWgU9FCg1k>(Snv{HO=j?V*1j>&7?H9tD*%d*Zt)slQyYsf&GFF4+J|p~L~7 zx{6Ibq=uDs^eWzo%2iO2fkSG3z?EA$mtYLaJ8-5R-b?J7*FsVawD^^e(b>SAHZk_? zXJa3FN0={%bX_`~iB{+6t+G*xs<{N^oCfISV^+TkRP(_3o|rbKt*W7dm1isaW5_!* z`EVxRs=}GP2YJuRdsxK}{^wQBS{r9q-U-+3Q9tE|KQN7cHj2wP-$2b(tFw@IZ&nL;(q&s5tq%s5d|-OGgeWEdq`GEDNaFwJaqKT30zJTN>m!FIZ} zU00vi+?HKUJ;qfmuUJF20CTSYp?&(um$`Sm=t%BCmlwH@Tyh^;y@dSaS+{w|(p@+P zAiPeW%g$jN!3E)lRda;`-~h@~aTs){pwI(8fN|>us1-|rmra&3pdLGPj@Wzb_b)-M zBn-egl%-q25MU+(iuf8qz(p9yMV9kQLgk$!&e`roLPfwqH(ZOe-YBo(R=+_j5kf|i|Y{%9m zYB2kOOYO>=N+h@R$biWc;Vjms9*o%|^Dh0W0SNFR(?kj?AMZa4bM~tiKnVVG8deg9 z?(K-f))6}lkJwvmDsh70U=Ag>YW~98W);ld2D`V*wt~t3m?^)UubR-9;0jaCAIraz z`*hB4p%tady+ZIGtgo3?3IA90M|zlO(@ch&iBf64wPEJ(kF<(T8w!gYd$k^1+Y7E0 z<5(AI`S9AFv-)PgvtSzAaRH;&`fJ#1(@65(NF~~yui@Lq&9ZnHYXWaQPZ=Gi@$M(E zIY0nZBsNYlfd@{@WT-z!Z{4vbd~hvN0U%uHOp=+#yWjZg#_jjNZM<4r>p9%sXaeD| z9F=hvVRiAaFg(S?q`Jl{CYwc4cAI!cwP--aTCmnYe)oC1Z}+y7-cEG#v3Iblf;5&) zIDlDuNK~dgn3?;6HT7qPAHeuSeho=~fUf5Zoyb5Fi8w&<4^}y!KtZ4I|4licu;-QY z3BiG|>Ss%l^O(WV2_oS2r!sFn=K6-MLp`msHmDsfje&2KPL!=O1bECHS^6|+LxtxD z3(AAc1tc+8>^tyTjRWxne)6#sy`X%MM5xwZI#0xH_J}>=kHjOVFD)JPRSW%>1~2Zz zi&h?h#$O@$!V?6nHj2jHZ$1W91RwF?zyB7qFT)E$?(kwIj`$Io$z0_FIvo!5jg8=3&> zlMj(~gr!oN{Drk1c+L%!P22juG9BgcO2v)|$0Jx0RZ~^;AaU*|XSHG#g@h41?}|sP z4-8Y{W54UvTmML6^NFkbUHJ&axa$(4@4YtG2~<3ZdU2L+>bRaJA5Y5FTiFPW#oMdz z#wFBT_r#fbsZ5DDSh0kxWc%!pA7RtrO;{v|Kx$IkjsxmA0ev7JkmhL_Mfz=O7n=In z@>8zAOl6DbLYIPa(NxTK4yhUUDI&n;^e=ekbI8_J15O_Ca(E^FD zkbAWf-YszPk2oyuYfW3S(qNrddiJ){sCigTxVR<&u$rNNk4oP|)^SbKA~RGYu_1#d zPGFp^q%^_r#MvbIy4EAzk#>`bM~(H1)SF!6Q>lNO%HO3fZZF2^0#@_!jF4L{-T>x< zq^lo7rg#~d$J4(-D+r>fcJ^QgVHANG zW;Iu^TA;T`W^P6$dP$Z+fvln^ywDM4^@?YD`UAAsfXb(kO6nbwAj7gdXE)Wj@EdQw zVO6t?b!nU=SJn`riS)7h5@odAomceNscgNL2kz5Pw@hraLL zRm@^l(s}?}+5n$+=&r%vNs=)N#Tu?3oLFnCnx+Asj3YpFvZ-+>J;YBP*9OCNd^KOj zJ4u8C%eNv5(e!W897LEpfpOz(yd4`KQFcjTw(;*q!vtYfuN#!Z@o5=ME80w;MoBlc z!Bf1IVQ->Onb7k@X;CNTMCZ{~dJo!c7x6Q!s3179M+K@R zaFBv~KleREMqK{|UiB!-Ir$M&CoCjCTD(KSJ%Dq9=wORYXN=r3>er^t(r)>mU4=LD z4<$Sj95P%uf_Vz{ajY(Y-XYkgL@~}?&=rY4-CE`T4OV>dK3Z!M0A}3EMdjf*8+OJq zW<;P4XJ2#p^D_pRk?xd!*%{MwGrorz8`A*MMa#wuKCN`%Q`Eq1yf>Ts3REhvb2Ski zv`R4u2CYFzY6h}2SA|Dl`BWwqJ#5>Ep|$%_)q)CoGeQSWRvAt*eT`lac{A>PI9^;h(&b|xL}(P@t_ZkP zyV37M1V|UeP_cWK(AF+kBC3O@@8P0KK;1}348Lz2Y7OX;=pNCg0%hC~)#c}vr8V=Z zNmLa7hW^E?$N~}YwnNdL4*(0@wrKNZ=n26ZsZHC(CC=YL@Z52epP{d`VAe$eN5c=< z5nEumdLR!YELc$)H??(HKpCI}h}=$EP{z@pzNNSgzUmV085OQ>D7wNy$lA@Sd%8#QJ(?5x0S(@xvNime1UuulC8e*}Fqc z;YUq<={DFj5WUm=g7NB0WP5K`= 1: + ret = paddle.max(self, *args, **kwargs), paddle.argmax(self, *args, **kwargs) + else: + ret = paddle.max(self, *args, **kwargs) + + return ret + +setattr(paddle.Tensor, "_max", _Tensor_max) + +def _Tensor_view(self, *args, **kwargs): + if args: + if len(args)==1 and isinstance(args[0], (tuple, list, str)): + return paddle.view(self, args[0]) + else: + return paddle.view(self, list(args)) + elif kwargs: + return paddle.view(self, shape_or_dtype = list(kwargs.values())[0]) + +setattr(paddle.Tensor, 'view', _Tensor_view) +############################## 相关utils函数，如上 ############################## + + + +@paddle.no_grad() +def quantize_weight_per_channel_absmax(w, n_bits=8): + # 获取每个channel的最大绝对值作为scale + scales = w.abs().max(keepdim=True, axis=-1)[0] + + # 计算量化的最大值 + q_max = 2 ** (n_bits - 1) - 1 + + # 关键修复：确保类型匹配 + # 方法1：将q_max转换为与scales相同的float类型 + scales = paddle.clip(scales, min=1e-05) / float(q_max) + + # 量化权重：除以scale，四舍五入，再乘回scale + w = paddle.round(w / scales) * scales + + return w + + +@paddle.no_grad() +def quantize_weight_per_tensor_absmax(w, n_bits=8): + scales = w.abs()._max() + q_max = 2 ** (n_bits - 1) - 1 + scales.clip_(min=1e-05).divide_(y=paddle.to_tensor(q_max)) + w.divide_(y=paddle.to_tensor(scales)).round_().multiply_(y=paddle.to_tensor(scales)) + return w + + +@paddle.no_grad() +def quantize_activation_per_token_absmax(t, n_bits=8): + t_shape = tuple(t.shape) + t.view(-1, t_shape[-1]) + scales = ( + t.abs().max(keepdim=True, axis=-1), + t.abs().argmax(keepdim=True, axis=-1), + )[0] + q_max = 2 ** (n_bits - 1) - 1 + scales.clip_(min=1e-05).divide_(y=paddle.to_tensor(q_max)) + t.divide_(y=paddle.to_tensor(scales)).round_().multiply_(y=paddle.to_tensor(scales)) + return t + + +@paddle.no_grad() +def quantize_activation_per_tensor_absmax(t, n_bits=8): + t_shape = tuple(t.shape) + t.view(-1, t_shape[-1]) + scales = t.abs()._max() + q_max = 2 ** (n_bits - 1) - 1 + scales.clip_(min=1e-05).divide_(y=paddle.to_tensor(q_max)) + t.divide_(y=paddle.to_tensor(scales)).round_().multiply_(y=paddle.to_tensor(scales)) + return t + + +class W8A8Linear(paddle.nn.Layer): + def __init__( + self, + in_features, + out_features, + bias=True, + act_quant="per_token", + quantize_output=False, + nbits=6, + ): + super().__init__() + self.nbits = nbits + self.in_features = in_features + self.out_features = out_features + out_0 = paddle.randn( + shape=[self.out_features, self.in_features], dtype="float16" + ) + out_0.stop_gradient = not False + self.register_buffer(name="weight", tensor=out_0) + if bias: + out_1 = paddle.zeros(shape=(1, self.out_features), dtype="float16") + out_1.stop_gradient = not False + self.register_buffer(name="bias", tensor=out_1) + else: + self.register_buffer(name="bias", tensor=None) + if act_quant == "per_token": + self.act_quant_name = "per_token" + self.act_quant = partial( + quantize_activation_per_token_absmax, n_bits=self.nbits + ) + elif act_quant == "per_tensor": + self.act_quant_name = "per_tensor" + self.act_quant = partial( + quantize_activation_per_tensor_absmax, n_bits=self.nbits + ) + else: + raise ValueError(f"Invalid act_quant: {act_quant}") + if quantize_output: + self.output_quant_name = self.act_quant_name + self.output_quant = self.act_quant + else: + self.output_quant_name = "None" + self.output_quant = lambda x: x + + def to(self, *args, **kwargs): + """Not Support auto convert *.to, please judge whether it is Pytorch API and convert by yourself""" + super(W8A8Linear, self).to(*args, **kwargs) + """Not Support auto convert *.to, please judge whether it is Pytorch API and convert by yourself""" + self.weight = self.weight.to(*args, **kwargs) + if self.bias is not None: + """Not Support auto convert *.to, please judge whether it is Pytorch API and convert by yourself""" + self.bias = self.bias.to(*args, **kwargs) + return self + + @paddle.no_grad() + def forward(self, x): + q_x = self.act_quant(x) + # wufazidong + # y = torch.functional.F.linear(q_x, self.weight, self.bias) + y = paddle.nn.functional.linear(q_x, self.weight, self.bias) + q_y = self.output_quant(y) + return q_y + + @staticmethod + def from_float( + module, + weight_quant="per_channel", + act_quant="per_token", + quantize_output=False, + nbits=8, + ): + assert isinstance(module, paddle.nn.Linear) + # import pdb; pdb.set_trace() + new_module = W8A8Linear( + module.weight.shape[0], + module.weight.shape[0], + module.bias is not None, + act_quant=act_quant, + quantize_output=quantize_output, + nbits=nbits, + ) + if weight_quant == "per_channel": + new_module.weight = quantize_weight_per_channel_absmax( + module.weight, n_bits=nbits + ) + elif weight_quant == "per_tensor": + new_module.weight = quantize_weight_per_tensor_absmax( + module.weight, n_bits=nbits + ) + else: + raise ValueError(f"Invalid weight_quant: {weight_quant}") + new_module.weight_quant_name = weight_quant + if module.bias is not None: + new_module.bias = module.bias + return new_module + + def __repr__(self): + return f"W8A8Linear({self.in_features}, {self.out_features}, bias={self.bias is not None}, weight_quant={self.weight_quant_name}, act_quant={self.act_quant_name}, output_quant={self.output_quant_name})" \ No newline at end of file diff --git a/example/JSQ/smoothquant/layerwrapper.py b/example/JSQ/smoothquant/layerwrapper.py new file mode 100644 index 000000000..33e59bdcd --- /dev/null +++ b/example/JSQ/smoothquant/layerwrapper.py @@ -0,0 +1,69 @@ +import copy + +import paddle +from smoothquant.fake_quant import W8A8Linear +from smoothquant.utils import clip_matrix + +############################## 相关utils函数，如下 ############################## + +def _Tensor_reshape(self, *args, **kwargs): + if args: + if len(args) == 1 and isinstance(args[0], (tuple, list)): + return paddle.reshape(self, args[0]) + else: + return paddle.reshape(self, list(args)) + elif kwargs: + assert "shape" in kwargs + return paddle.reshape(self, shape=kwargs["shape"]) + +setattr(paddle.Tensor, "reshape", _Tensor_reshape) +############################## 相关utils函数，如上 ############################## + + + +class WrappedGPT: + """ + This class wraps a GPT layer for specific operations. + """ + + def __init__(self, layer, layer_id=0, layer_name="none"): + self.layer = layer + self.dev = self.layer.weight.place + self.rows = tuple(layer.weight.data.shape)[1] + self.columns = tuple(layer.weight.data.shape)[0] + self.inp_sum = None + self.inp_num = 0 + self.scaler_row = paddle.zeros(shape=self.columns) + self.nsamples = 0 + self.layer_id = layer_id + self.layer_name = layer_name + + def add_batch(self, inp, out): + if len(tuple(inp.shape)) == 2: + inp = inp.unsqueeze(axis=0) + tmp = tuple(inp.shape)[0] + if isinstance(self.layer, paddle.nn.Linear) or isinstance( + self.layer, W8A8Linear + ): + if len(tuple(inp.shape)) == 3: + inp = inp.reshape((-1, tuple(inp.shape)[-1])) + inp = inp.t() + if self.inp_sum is None: + self.inp_sum = copy.deepcopy(inp.t()) + else: + self.inp_sum += copy.deepcopy(inp.t()) + self.inp_num += 1 + self.scaler_row *= self.nsamples / (self.nsamples + tmp) + self.nsamples += tmp + inp = inp.astype("float32") + # self.scaler_row += paddle.linalg.norm(x=inp, p=2, axis=1) ** 2 / self.nsamples + # self.scaler_row += torch.norm(inp, p=2, dim=1) ** 2 / self.nsamples + # 用 elementwise add（避免 linalg.norm 的潜在兼容问题） + # import pdb + # import pdb; pdb.set_trace() + self.scaler_row = paddle.add(self.scaler_row, paddle.sum(inp * inp, axis=1) / float(self.nsamples)) + # self.scaler_row = self.scaler_row + paddle.sum(inp * inp, axis=1) / float(self.nsamples) + + + + diff --git a/example/JSQ/smoothquant/quantize.py b/example/JSQ/smoothquant/quantize.py new file mode 100644 index 000000000..2e49b2268 --- /dev/null +++ b/example/JSQ/smoothquant/quantize.py @@ -0,0 +1,195 @@ +import paddle +import paddlenlp +from smoothquant.fake_quant import W8A8Linear +# from smoothquant.modeling_chatglm import GLMBlock + + +@paddle.no_grad() +def quantize_model( + model, weight_quant="per_tensor", act_quant="per_tensor", quantize_bmm_input=True +): + for name, m in model.named_sublayers(include_self=True): + # wufazidong + # if isinstance(m, transformers.models.opt.modeling_opt.OPTDecoderLayer): + if isinstance(m, paddlenlp.transformers.opt.modeling_opt.OPTDecoderLayer): + m.fc1 = W8A8Linear.from_float( + m.fc1, weight_quant=weight_quant, act_quant=act_quant + ) + m.fc2 = W8A8Linear.from_float( + m.fc2, weight_quant=weight_quant, act_quant=act_quant + ) + # wufazidong + # elif isinstance(m, transformers.models.opt.modeling_opt.OPTAttention): + elif isinstance(m, paddlenlp.transformers.opt.modeling_opt.OPTAttention): + m.q_proj = W8A8Linear.from_float( + m.q_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.k_proj = W8A8Linear.from_float( + m.k_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.v_proj = W8A8Linear.from_float( + m.v_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.out_proj = W8A8Linear.from_float( + m.out_proj, weight_quant=weight_quant, act_quant=act_quant + ) + # wufazidong + # elif isinstance(m, transformers.models.llama.modeling_llama.LlamaAttention): + elif isinstance(m, paddlenlp.transformers.LlamaAttention): + m.q_proj = W8A8Linear.from_float( + m.q_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.k_proj = W8A8Linear.from_float( + m.k_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.v_proj = W8A8Linear.from_float( + m.v_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + ) + m.o_proj = W8A8Linear.from_float( + m.o_proj, weight_quant=weight_quant, act_quant=act_quant + ) + # wufazidong + # elif isinstance(m, transformers.models.llama.modeling_llama.LlamaMLP): + # 注意：这里假设LlamaMLP在PaddleNLP的LlamaForCausalLM模型中有类似的定义， + # 实际使用时需要确认PaddleNLP中是否有对应的类或模块，并可能需要调整导入路径。 + elif isinstance(m, paddlenlp.transformers.LlamaForCausalLM.LlamaMLP): + m.gate_proj = W8A8Linear.from_float( + m.gate_proj, weight_quant=weight_quant, act_quant=act_quant + ) + m.down_proj = W8A8Linear.from_float( + m.down_proj, weight_quant=weight_quant, act_quant=act_quant + ) + m.up_proj = W8A8Linear.from_float( + m.up_proj, weight_quant=weight_quant, act_quant=act_quant + ) + return model + + +@paddle.no_grad() +def quantize_layer( + module, + nbits, + weight_quant="per_channel", + act_quant="per_token", + quantize_bmm_input=True, +): + for name, m in module.named_sublayers(include_self=True): + # wufazidong + # if isinstance(m, transformers.models.opt.modeling_opt.OPTDecoderLayer): + if isinstance(m, paddlenlp.transformers.llama.modeling.LlamaAttention): + m.q_proj = W8A8Linear.from_float( + m.q_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + nbits=nbits, + ) + m.k_proj = W8A8Linear.from_float( + m.k_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + nbits=nbits, + ) + m.v_proj = W8A8Linear.from_float( + m.v_proj, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + nbits=nbits, + ) + m.o_proj = W8A8Linear.from_float( + m.o_proj, weight_quant=weight_quant, act_quant=act_quant, nbits=nbits + ) + # wufazidong + # elif isinstance(m, transformers.models.llama.modeling_llama.LlamaDecoderLayer): + elif isinstance(m, paddlenlp.transformers.llama.modeling.LlamaDecoderLayer): + m.mlp.gate_proj = W8A8Linear.from_float( + m.mlp.gate_proj, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + m.mlp.down_proj = W8A8Linear.from_float( + m.mlp.down_proj, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + m.mlp.up_proj = W8A8Linear.from_float( + m.mlp.up_proj, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + elif "FalconDecoderLayer" in m.__class__.__name__: + m.self_attention.query_key_value = W8A8Linear.from_float( + m.self_attention.query_key_value, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_output=quantize_bmm_input, + nbits=nbits, + ) + m.self_attention.dense = W8A8Linear.from_float( + m.self_attention.dense, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + m.mlp.dense_h_to_4h = W8A8Linear.from_float( + m.mlp.dense_h_to_4h, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + m.mlp.dense_4h_to_h = W8A8Linear.from_float( + m.mlp.dense_4h_to_h, + weight_quant=weight_quant, + act_quant=act_quant, + nbits=nbits, + ) + # elif isinstance(m, GLMBlock) or "GLMBlock" in m.__class__.__name__: + # m.self_attention.query_key_value = W8A8Linear.from_float( + # m.self_attention.query_key_value, + # weight_quant=weight_quant, + # act_quant=act_quant, + # quantize_output=quantize_bmm_input, + # nbits=nbits, + # ) + # m.self_attention.dense = W8A8Linear.from_float( + # m.self_attention.dense, + # weight_quant=weight_quant, + # act_quant=act_quant, + # nbits=nbits, + # ) + # m.mlp.dense_h_to_4h = W8A8Linear.from_float( + # m.mlp.dense_h_to_4h, + # weight_quant=weight_quant, + # act_quant=act_quant, + # nbits=nbits, + # ) + # m.mlp.dense_4h_to_h = W8A8Linear.from_float( + # m.mlp.dense_4h_to_h, + # weight_quant=weight_quant, + # act_quant=act_quant, + # nbits=nbits, + # ) + return module \ No newline at end of file diff --git a/example/JSQ/smoothquant/smooth.py b/example/JSQ/smoothquant/smooth.py new file mode 100644 index 000000000..1edf2ff28 --- /dev/null +++ b/example/JSQ/smoothquant/smooth.py @@ -0,0 +1,155 @@ +import paddle +import paddlenlp +# from smoothquant.modeling_chatglm import RMSNorm + +############################## 相关utils函数，如下 ############################## + +def _Tensor_view(self, *args, **kwargs): + if args: + if len(args)==1 and isinstance(args[0], (tuple, list, str)): + return paddle.view(self, args[0]) + else: + return paddle.view(self, list(args)) + elif kwargs: + return paddle.view(self, shape_or_dtype = list(kwargs.values())[0]) + +setattr(paddle.Tensor, 'view', _Tensor_view) +############################## 相关utils函数，如上 ############################## + + + +@paddle.no_grad() +def smooth_ln_fcs(ln, fcs, act_scales, alpha=0.5): + if not isinstance(fcs, list): + fcs = [fcs] + assert ( + isinstance(ln, paddle.nn.LayerNorm) + # wufazidong + # or isinstance(ln, transformers.models.llama.modeling_llama.LlamaRMSNorm) + or isinstance(ln, paddlenlp.transformers.llama.modeling.LlamaRMSNorm) + # or isinstance(ln, RMSNorm) + or "RMSNorm" in ln.__class__.__name__ + ) + # for fc in fcs: + # assert isinstance(fc, paddle.nn.Linear) + # assert ln.weight.size == fc.in_features == act_scales.size + device, dtype = fcs[0].weight.place, fcs[0].weight.dtype + act_scales = act_scales.to(device=device, dtype=dtype) + # import pdb; pdb.set_trace() + weight_scales = paddle.concat( + x=[ + ( + fc.weight.transpose([1,0]).abs().max(keepdim=True, axis=0), + fc.weight.transpose([1,0]).abs().argmax(keepdim=True, axis=0), + )[0] + for fc in fcs + ], + axis=0, + ) + weight_scales = (weight_scales.max(axis=0), weight_scales.argmax(axis=0))[0].clip( + min=1e-05 + ) + + scales = ( + (act_scales.pow(y=alpha) / weight_scales.pow(y=1 - alpha)) + .clip(min=1e-05) + .to(device) + .to(dtype) + ) + ln.weight.divide_(y=paddle.to_tensor(scales)) + if hasattr(ln, "bias"): + ln.bias.divide_(y=paddle.to_tensor(scales)) + for fc in fcs: + fc.weight.transpose_([1,0]).multiply_(y=paddle.to_tensor(scales.view(1, -1))).transpose_([1,0]) + + +@paddle.no_grad() +def smooth_lm(model, scales, alpha=0.5): + for name, module in model.named_sublayers(include_self=True): + # wufazidong + # if isinstance(module, transformers.models.opt.modeling_opt.OPTDecoderLayer): + if isinstance(module, paddlenlp.transformers.OPTDecoderLayer): + attn_ln = module.self_attn_layer_norm + qkv = [ + module.self_attn.q_proj, + module.self_attn.k_proj, + module.self_attn.v_proj, + ] + qkv_input_scales = scales[name + ".self_attn.q_proj"] + smooth_ln_fcs(attn_ln, qkv, qkv_input_scales, alpha) + ffn_ln = module.final_layer_norm + fc1 = module.fc1 + fc1_input_scales = scales[name + ".fc1"] + smooth_ln_fcs(ffn_ln, fc1, fc1_input_scales, alpha) + # wufazidong + # elif isinstance( + # module, transformers.models.llama.modeling_llama.LlamaDecoderLayer + # ): + elif isinstance( + module, paddlenlp.transformers.llama.modeling_llama.LlamaDecoderLayer + ): + attn_ln = module.input_layernorm + qkv = [ + module.self_attn.q_proj, + module.self_attn.k_proj, + module.self_attn.v_proj, + ] + if "layer" in name: + qkv_input_scales = scales[name + ".self_attn.q_proj"] + else: + qkv_input_scales = scales["self_attn.q_proj"] + smooth_ln_fcs(attn_ln, qkv, qkv_input_scales, alpha) + ffn_ln = module.post_attention_layernorm + fc1 = [module.mlp.gate_proj, module.mlp.up_proj] + if "layer" in name: + fc1_input_scales = scales[name + ".mlp.up_proj"] + else: + fc1_input_scales = scales["mlp.up_proj"] + smooth_ln_fcs(ffn_ln, fc1, fc1_input_scales, alpha) + # wufazidong + # elif isinstance(module, transformers.models.bloom.modeling_bloom.BloomBlock): + elif isinstance(module, paddlenlp.transformers.bloom.modeling_bloom.BloomBlock): + attn_ln = module.input_layernorm + qkv = module.self_attention.query_key_value + qkv_input_scales = scales[name + ".self_attention.query_key_value"] + smooth_ln_fcs(attn_ln, qkv, qkv_input_scales, alpha) + ffn_ln = module.post_attention_layernorm + fc1 = module.mlp.dense_h_to_4h + fc1_input_scales = scales[name + ".mlp.dense_h_to_4h"] + smooth_ln_fcs(ffn_ln, fc1, fc1_input_scales, alpha) + + +@paddle.no_grad() +def smooth_layer(name, module, scales, alpha=0.5): + # wufazidong + # if isinstance(module, transformers.models.llama.modeling_llama.LlamaDecoderLayer): + if isinstance(module, paddlenlp.transformers.llama.modeling.LlamaDecoderLayer): + # 注意：这里假设PaddleNLP中有对应的LlamaDecoderLayer类，实际使用时需要确认PaddleNLP的版本和API。 + attn_ln = module.input_layernorm + qkv = [ + module.self_attn.q_proj, + module.self_attn.k_proj, + module.self_attn.v_proj, + ] + qkv_input_scales = scales[name + ".self_attn.q_proj"] + smooth_ln_fcs(attn_ln, qkv, qkv_input_scales, alpha) + ffn_ln = module.post_attention_layernorm + fc1 = [module.mlp.gate_proj, module.mlp.up_proj] + fc1_input_scales = scales[name + ".mlp.gate_proj"] + smooth_ln_fcs(ffn_ln, fc1, fc1_input_scales, alpha) + elif "FalconDecoderLayer" in module.__class__.__name__: + attn_ln = module.input_layernorm + qkv_fc1 = [module.self_attention.query_key_value, module.mlp.dense_h_to_4h] + qkv_input_scales = scales[name + ".self_attention.query_key_value"] + smooth_ln_fcs(attn_ln, qkv_fc1, qkv_input_scales, alpha) + elif "GLMBlock" in module.__class__.__name__: + attn_ln = module.input_layernorm + qkv = [module.self_attention.query_key_value] + qkv_input_scales = scales[name + ".self_attention.query_key_value"] + smooth_ln_fcs(attn_ln, qkv, qkv_input_scales, alpha) + ffn_ln = module.post_attention_layernorm + fc1 = [module.mlp.dense_h_to_4h] + fc1_input_scales = scales[name + ".mlp.dense_h_to_4h"] + smooth_ln_fcs(ffn_ln, fc1, fc1_input_scales, alpha) + else: + raise TypeError(f"未添加的Decoder类{module}") \ No newline at end of file diff --git a/example/JSQ/smoothquant/utils.py b/example/JSQ/smoothquant/utils.py new file mode 100644 index 000000000..53125e0a3 --- /dev/null +++ b/example/JSQ/smoothquant/utils.py @@ -0,0 +1,276 @@ +import copy + +import paddle +from smoothquant.fake_quant import W8A8Linear + +############################## 相关utils函数，如下 ############################## + +def dim2perm(ndim, dim0, dim1): + perm = list(range(ndim)) + perm[dim0], perm[dim1] = perm[dim1], perm[dim0] + return perm + +def _Tensor_reshape(self, *args, **kwargs): + if args: + if len(args) == 1 and isinstance(args[0], (tuple, list)): + return paddle.reshape(self, args[0]) + else: + return paddle.reshape(self, list(args)) + elif kwargs: + assert "shape" in kwargs + return paddle.reshape(self, shape=kwargs["shape"]) + +setattr(paddle.Tensor, "reshape", _Tensor_reshape) +############################## 相关utils函数，如上 ############################## + + + +def clip_matrix(matrix, abs=True, clip_l=0, clip_h=0, channel=False): + if clip_l == 0 and clip_h == 0: + return matrix + if channel: + matrix_flatten = matrix + if abs: + matrix_flatten = paddle.abs(x=matrix) + max_threshold = None + min_threshold = None + if clip_h != 0: + max_threshold = paddle.quantile( + x=matrix_flatten[0].astype(dtype="float64"), q=1 - clip_h, axis=0 + ) + clipped_matrix = paddle.clip(x=matrix, min=-max_threshold, max=max_threshold) + return clipped_matrix + else: + num_elements = matrix.size + if abs: + matrix_flatten = paddle.abs(x=matrix).flatten() + else: + matrix_flatten = matrix.flatten() + max_threshold = None + min_threshold = None + if clip_l != 0: + low_index = int(clip_l * num_elements) + min_threshold, _ = paddle.topk(x=matrix_flatten, largest=False, k=low_index) + min_threshold = min_threshold[-1] + if clip_h != 0: + high_index = int(clip_h * num_elements) + max_threshold, _ = paddle.topk(x=matrix_flatten, largest=True, k=high_index) + max_threshold = max_threshold[-1] + if abs: + clipped_matrix = paddle.clip( + x=matrix, min=-max_threshold, max=max_threshold + ) + else: + clipped_matrix = paddle.clip(x=matrix, min=min_threshold, max=max_threshold) + return clipped_matrix + + +def find_layers(module, layers=[paddle.nn.Linear, W8A8Linear], name=""): + if type(module) in layers or "FalconLinear" in module.__class__.__name__: + return {name: module} + else: + pass + res = {} + for name1, child in module.named_children(): + res.update( + find_layers( + child, layers=layers, name=name + "." + name1 if name != "" else name1 + ) + ) + return res + + +def check_sparsity(model): + CHATGLM = False + Falcon = False + if hasattr(model, "transformer"): + if hasattr(model.transformer, "embedding"): + CHATGLM = True + elif hasattr(model.transformer, "word_embeddings"): + Falcon = True + if CHATGLM: + layers = model.transformer.encoder.layers + elif Falcon: + layers = model.transformer.h + else: + layers = model.llama.layers + count = 0 + total_params = 0 + for i in range(len(layers)): + layer = layers[i] + subset = find_layers(layer) + sub_count = 0 + sub_params = 0 + for name in subset: + W = subset[name].weight.data + count += (W == 0).sum().item() + total_params += W.size + sub_count += (W == 0).sum().item() + sub_params += W.size + print(f"layer {i} sparsity {float(sub_count) / sub_params:.6f}") + return float(count) / total_params + + +def prepare_calibration_input(model, dataloader, device): + CHATGLM = False + Falcon = False + if hasattr(model, "transformer"): + if hasattr(model.transformer, "embedding"): + CHATGLM = True + elif hasattr(model.transformer, "word_embeddings"): + Falcon = True + use_cache = model.config.use_cache + model.config.use_cache = False + if CHATGLM: + layers = model.transformer.encoder.layers + elif Falcon: + layers = model.transformer.h + else: + layers = model.llama.layers + # if "model.embed_tokens" in model.hf_device_map: + # device = model.hf_device_map["model.embed_tokens"] + device = model.llama.embed_tokens.weight.place + dtype = next(iter(model.parameters())).dtype + inps = paddle.zeros( + shape=(128, model.seqlen, model.config.hidden_size), dtype=dtype + ) + inps.stop_gradient = not False + cache = {"i": 0, "attention_mask": None, "position_ids": None} + + class Catcher(paddle.nn.Layer): + def __init__(self, module): + super().__init__() + self.module = module + + def forward(self, inp, *args, **kwargs): + if CHATGLM: + inps[cache["i"]] = inp.transpose(perm=dim2perm(inp.ndim, 0, 1))[0] + else: + inps[cache["i"]] = inp + cache["i"] += 1 + if CHATGLM: + cache["attention_mask"] = args[0] + else: + # cache["attention_mask"] = kwargs["attention_mask"] + cache["attention_mask"] = kwargs.get("attention_mask", None) + if CHATGLM: + cache["position_ids"] = args[1] + elif Falcon: + pass + else: + # cache["position_ids"] = kwargs["position_ids"] + cache["position_ids"] = kwargs.get("position_ids", None) + raise ValueError + + layers[0] = Catcher(layers[0]) + for batch in dataloader: + try: + if cache["i"] == 0: + model_inputs = model.prepare_inputs_for_generation(batch[0].to(device)) + if "position_ids" in model_inputs: + cache["position_ids"] = model_inputs["position_ids"] + else: + cache["position_ids"] = paddle.arange( + start=0, end=tuple(batch[0].shape)[1], dtype="int64" + ).unsqueeze(axis=0) + model(batch[0].to(device)) + except ValueError: + pass + layers[0] = layers[0].module + outs = paddle.zeros_like(x=inps) + attention_mask = cache["attention_mask"] + position_ids = cache["position_ids"] + model.config.use_cache = use_cache + return inps, outs, attention_mask, position_ids + + +def return_given_alpha(alpha, sort_res, W_metric, tmp_metric, sum_before): + thres_cumsum = sum_before * alpha + sort_mask = tmp_metric <= thres_cumsum.reshape((-1, 1)) + thres = paddle.take_along_axis( + arr=sort_res[0], + axis=1, + indices=sort_mask.sum(dim=1, keepdims=True) - 1, + broadcast=False, + ) + W_mask = W_metric <= thres + cur_sparsity = (W_mask == True).sum() / W_mask.size + return W_mask, cur_sparsity + + +def cal_mse_layer( + args, layer1, layer2, inps, attention_mask, position_ids, outs1=None, outs2=None +): + if outs1 is None: + layer1_outs = [] + for i in range(args.nsamples): + with paddle.no_grad(): + layer1_outs.append( + layer1( + inps[i].unsqueeze(axis=0), + attention_mask=attention_mask, + position_ids=position_ids, + )[0][0] + ) + else: + layer1_outs = outs1 + if outs2 is None: + layer2_outs = [] + for i in range(args.nsamples): + with paddle.no_grad(): + layer2_outs.append( + layer2( + inps[i].unsqueeze(axis=0), + attention_mask=attention_mask, + position_ids=position_ids, + )[0][0] + ) + else: + layer2_outs = outs2 + mse = 0.0 + print(tuple(layer1_outs[0].shape)) + for i in range(args.nsamples): + device = layer2_outs[i].place + mse += paddle.nn.functional.mse_loss( + input=layer1_outs[i].to(device), label=layer2_outs[i] + ).item() + return mse + + +def generate_ss(activation, weight): + # weight: (out_features, in_features) for Paddle + cout, cin = weight.shape + ss = paddle.zeros_like(weight) + + # 确保 activation 是 2D + if len(activation.shape) == 1: + activation = activation.unsqueeze(0) + for i in range(cout): + w = weight.clone() + w[i, :] = 0 + + # w: (cout, cin), 需要转置为 (cin, cout) + # activation: (batch, cin) @ w.T(cin, cout) = (batch, cout) + # w_t = w.transpose([1, 0]) # 显式转置 + out = paddle.matmul(activation, w) + + max_values = paddle.max(out, axis=0) + min_values = paddle.min(out, axis=0) + + ss[i, :] = max_values - min_values + + ss = paddle.where( + paddle.isinf(ss), + paddle.to_tensor(100.0, dtype=ss.dtype), + ss + ) + ss = ss.transpose([1, 0]) + return ss + + +def generate_ss2(activation, weight): + out = activation @ weight.t() + max_values, _ = paddle.max(x=out, axis=0), paddle.argmax(x=out, axis=0) + min_values, _ = paddle.min(x=out, axis=0), paddle.argmin(x=out, axis=0) + row_ss = (max_values - min_values).reshape((-1, 1)) + return row_ss \ No newline at end of file From 0292248b59360d81f6b451934398d0904b2a7c60 Mon Sep 17 00:00:00 2001 From: Yuanming Chen <122955822+CYMCharming@users.noreply.github.com> Date: Sat, 14 Mar 2026 19:25:16 +0800 Subject: [PATCH 2/2] Fix README title Fix the README title --- example/JSQ/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/JSQ/README.md b/example/JSQ/README.md index 17a6fd997..cdb022893 100644 --- a/example/JSQ/README.md +++ b/example/JSQ/README.md @@ -1,7 +1,7 @@ -## FIMA-Q: Post-Training Quantization for Vision Transformers by Fisher Information Matrix Approximation +## Compressing Large Language Models by Joint Sparsification and Quantization ## 1. 简介 本示例介绍了一种剪枝量化联合压缩方法。 -技术详情见论文 [Compressing Large Language Models by Joint Sparsification and Quantization](https://openreview.net/attachment?id=sCGRhnuMUJ&name=pdf) \ No newline at end of file +技术详情见论文 [Compressing Large Language Models by Joint Sparsification and Quantization](https://openreview.net/attachment?id=sCGRhnuMUJ&name=pdf)