From f18162a2ddbd19538dcafc32bcd1a12b6be5d976 Mon Sep 17 00:00:00 2001 From: darealshinji Date: Fri, 24 Jul 2026 13:55:29 +0200 Subject: [PATCH 1/2] x86 ASM: translate from MASM to NASM dialect --- Asm/x86/7zAsm.asm | 341 ----------- Asm/x86/7zAsm.inc | 368 +++++++++++ Asm/x86/7zCrcOpt.asm | 285 +++++---- Asm/x86/AesOpt.asm | 794 +++++++++++------------- Asm/x86/LzFindOpt.asm | 489 +++++++-------- Asm/x86/LzmaDecOpt.asm | 1260 +++++++++++++++++++------------------- Asm/x86/Sha1Opt.asm | 313 ++++------ Asm/x86/Sha256Opt.asm | 296 ++++----- Asm/x86/Sort.asm | 715 ++++++++++----------- Asm/x86/XzCrc64Opt.asm | 521 ++++++++-------- C/7zip_gcc_c.mak | 18 +- CPP/7zip/7zip_gcc.mak | 26 +- CPP/7zip/LzmaDec_gcc.mak | 6 + CPP/Build.mak | 9 +- 14 files changed, 2621 insertions(+), 2820 deletions(-) delete mode 100644 Asm/x86/7zAsm.asm create mode 100644 Asm/x86/7zAsm.inc diff --git a/Asm/x86/7zAsm.asm b/Asm/x86/7zAsm.asm deleted file mode 100644 index 8910d16c6..000000000 --- a/Asm/x86/7zAsm.asm +++ /dev/null @@ -1,341 +0,0 @@ -; 7zAsm.asm -- ASM macros -; 2023-12-08 : Igor Pavlov : Public domain - - -; UASM can require these changes -; OPTION FRAMEPRESERVEFLAGS:ON -; OPTION PROLOGUE:NONE -; OPTION EPILOGUE:NONE - -ifdef @wordsize -; @wordsize is defined only in JWASM and ASMC and is not defined in MASM -; @wordsize eq 8 for 64-bit x64 -; @wordsize eq 2 for 32-bit x86 -if @wordsize eq 8 - x64 equ 1 -endif -else -ifdef RAX - x64 equ 1 -endif -endif - - -ifdef x64 - IS_X64 equ 1 -else - IS_X64 equ 0 -endif - -ifdef ABI_LINUX - IS_LINUX equ 1 -else - IS_LINUX equ 0 -endif - -ifndef x64 -; Use ABI_CDECL for x86 (32-bit) only -; if ABI_CDECL is not defined, we use fastcall abi -ifdef ABI_CDECL - IS_CDECL equ 1 -else - IS_CDECL equ 0 -endif -endif - -OPTION PROLOGUE:NONE -OPTION EPILOGUE:NONE - -MY_ASM_START macro - ifdef x64 - .code - else - .386 - .model flat - _TEXT$00 SEGMENT PARA PUBLIC 'CODE' - endif -endm - -MY_PROC macro name:req, numParams:req - align 16 - proc_numParams = numParams - if (IS_X64 gt 0) - proc_name equ name - elseif (IS_LINUX gt 0) - proc_name equ name - elseif (IS_CDECL gt 0) - proc_name equ @CatStr(_,name) - else - proc_name equ @CatStr(@,name,@, %numParams * 4) - endif - proc_name PROC -endm - -MY_ENDP macro - if (IS_X64 gt 0) - ret - elseif (IS_CDECL gt 0) - ret - elseif (proc_numParams LT 3) - ret - else - ret (proc_numParams - 2) * 4 - endif - proc_name ENDP -endm - - -ifdef x64 - REG_SIZE equ 8 - REG_LOGAR_SIZE equ 3 -else - REG_SIZE equ 4 - REG_LOGAR_SIZE equ 2 -endif - - x0 equ EAX - x1 equ ECX - x2 equ EDX - x3 equ EBX - x4 equ ESP - x5 equ EBP - x6 equ ESI - x7 equ EDI - - x0_W equ AX - x1_W equ CX - x2_W equ DX - x3_W equ BX - - x5_W equ BP - x6_W equ SI - x7_W equ DI - - x0_L equ AL - x1_L equ CL - x2_L equ DL - x3_L equ BL - - x0_H equ AH - x1_H equ CH - x2_H equ DH - x3_H equ BH - -; r0_L equ AL -; r1_L equ CL -; r2_L equ DL -; r3_L equ BL - -; r0_H equ AH -; r1_H equ CH -; r2_H equ DH -; r3_H equ BH - - -ifdef x64 - x5_L equ BPL - x6_L equ SIL - x7_L equ DIL - x8_L equ r8b - x9_L equ r9b - x10_L equ r10b - x11_L equ r11b - x12_L equ r12b - x13_L equ r13b - x14_L equ r14b - x15_L equ r15b - - r0 equ RAX - r1 equ RCX - r2 equ RDX - r3 equ RBX - r4 equ RSP - r5 equ RBP - r6 equ RSI - r7 equ RDI - x8 equ r8d - x9 equ r9d - x10 equ r10d - x11 equ r11d - x12 equ r12d - x13 equ r13d - x14 equ r14d - x15 equ r15d -else - r0 equ x0 - r1 equ x1 - r2 equ x2 - r3 equ x3 - r4 equ x4 - r5 equ x5 - r6 equ x6 - r7 equ x7 -endif - - x0_R equ r0 - x1_R equ r1 - x2_R equ r2 - x3_R equ r3 - x4_R equ r4 - x5_R equ r5 - x6_R equ r6 - x7_R equ r7 - x8_R equ r8 - x9_R equ r9 - x10_R equ r10 - x11_R equ r11 - x12_R equ r12 - x13_R equ r13 - x14_R equ r14 - x15_R equ r15 - -ifdef x64 -ifdef ABI_LINUX - -MY_PUSH_2_REGS macro - push r3 - push r5 -endm - -MY_POP_2_REGS macro - pop r5 - pop r3 -endm - -endif -endif - - -MY_PUSH_4_REGS macro - push r3 - push r5 - push r6 - push r7 -endm - -MY_POP_4_REGS macro - pop r7 - pop r6 - pop r5 - pop r3 -endm - - -; for fastcall and for WIN-x64 -REG_PARAM_0_x equ x1 -REG_PARAM_0 equ r1 -REG_PARAM_1_x equ x2 -REG_PARAM_1 equ r2 - -ifndef x64 -; for x86-fastcall - -REG_ABI_PARAM_0_x equ REG_PARAM_0_x -REG_ABI_PARAM_0 equ REG_PARAM_0 -REG_ABI_PARAM_1_x equ REG_PARAM_1_x -REG_ABI_PARAM_1 equ REG_PARAM_1 - -MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 macro - MY_PUSH_4_REGS -endm - -MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 macro - MY_POP_4_REGS -endm - -else -; x64 - -if (IS_LINUX eq 0) - -; for WIN-x64: -REG_PARAM_2_x equ x8 -REG_PARAM_2 equ r8 -REG_PARAM_3 equ r9 - -REG_ABI_PARAM_0_x equ REG_PARAM_0_x -REG_ABI_PARAM_0 equ REG_PARAM_0 -REG_ABI_PARAM_1_x equ REG_PARAM_1_x -REG_ABI_PARAM_1 equ REG_PARAM_1 -REG_ABI_PARAM_2_x equ REG_PARAM_2_x -REG_ABI_PARAM_2 equ REG_PARAM_2 -REG_ABI_PARAM_3 equ REG_PARAM_3 - -else -; for LINUX-x64: -REG_LINUX_PARAM_0_x equ x7 -REG_LINUX_PARAM_0 equ r7 -REG_LINUX_PARAM_1_x equ x6 -REG_LINUX_PARAM_1 equ r6 -REG_LINUX_PARAM_2 equ r2 -REG_LINUX_PARAM_3 equ r1 -REG_LINUX_PARAM_4_x equ x8 -REG_LINUX_PARAM_4 equ r8 -REG_LINUX_PARAM_5 equ r9 - -REG_ABI_PARAM_0_x equ REG_LINUX_PARAM_0_x -REG_ABI_PARAM_0 equ REG_LINUX_PARAM_0 -REG_ABI_PARAM_1_x equ REG_LINUX_PARAM_1_x -REG_ABI_PARAM_1 equ REG_LINUX_PARAM_1 -REG_ABI_PARAM_2 equ REG_LINUX_PARAM_2 -REG_ABI_PARAM_3 equ REG_LINUX_PARAM_3 -REG_ABI_PARAM_4_x equ REG_LINUX_PARAM_4_x -REG_ABI_PARAM_4 equ REG_LINUX_PARAM_4 -REG_ABI_PARAM_5 equ REG_LINUX_PARAM_5 - -MY_ABI_LINUX_TO_WIN_2 macro - mov r2, r6 - mov r1, r7 -endm - -MY_ABI_LINUX_TO_WIN_3 macro - mov r8, r2 - mov r2, r6 - mov r1, r7 -endm - -MY_ABI_LINUX_TO_WIN_4 macro - mov r9, r1 - mov r8, r2 - mov r2, r6 - mov r1, r7 -endm - -endif ; IS_LINUX - - -MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 macro - if (IS_LINUX gt 0) - MY_PUSH_2_REGS - else - MY_PUSH_4_REGS - endif -endm - -MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 macro - if (IS_LINUX gt 0) - MY_POP_2_REGS - else - MY_POP_4_REGS - endif -endm - - -MY_PUSH_PRESERVED_ABI_REGS macro - MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 - push r12 - push r13 - push r14 - push r15 -endm - - -MY_POP_PRESERVED_ABI_REGS macro - pop r15 - pop r14 - pop r13 - pop r12 - MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 -endm - -endif ; x64 diff --git a/Asm/x86/7zAsm.inc b/Asm/x86/7zAsm.inc new file mode 100644 index 000000000..fc957f1f3 --- /dev/null +++ b/Asm/x86/7zAsm.inc @@ -0,0 +1,368 @@ +; 7zAsm.asm -- ASM macros +; 2023-12-08 : Igor Pavlov : Public domain +; 2026-07-29 : Carsten Janssen : x86 ASM files were translated from MASM into NASM dialect + + +%define XBITS __?BITS?__ + +%define WINDOWS 1 +%define LINUX 2 + +%ifidni __?OUTPUT_FORMAT?__, win32 + %define ABI WINDOWS +%elifidni __?OUTPUT_FORMAT?__, win64 + %define ABI WINDOWS +%else + %define ABI LINUX +%endif + +; Use ABI_CDECL for x86 (32-bit) only +; if ABI_CDECL is not defined, we use fastcall ABI +%if (XBITS == 32) && (ABI == WINDOWS) + %ifdef ABI_CDECL + %define IS_CDECL 1 + %endif +%endif +%ifndef IS_CDECL + %define IS_CDECL 0 +%endif +%if IS_CDECL == 1 + %pragma win gprefix _ +%endif + +%macro MY_ASM_START 0 + default rel + %use smartalign + %if XBITS == 32 + [section .text$00 align=16 exec] + %else + [section .text] + %endif +%endmacro + +%if ABI == LINUX + %define READONLY .rodata +%else + %define READONLY .rdata +%endif + +%macro GLOBAL_FUNC 1 + %if ABI == LINUX + global %1: function + %else + global %1 + %endif + %1: +%endmacro + +%macro MY_PROC 2 + ALIGN 16 + %assign proc_numParams %2 + %if (XBITS == 64) || (ABI == LINUX) || (IS_CDECL == 1) + GLOBAL_FUNC %1 + %else + GLOBAL_FUNC %tok(%strcat(@, %1, @, %eval(%2 * 4))) + %endif +%endmacro + +%macro MY_ENDP 0 + %if (XBITS == 64) || (IS_CDECL == 1) || (proc_numParams < 3) + ret + %else + ret (proc_numParams - 2) * 4 + %endif +%endmacro + + +%if XBITS == 64 + %define REG_SIZE 8 + %define REG_LOGAR_SIZE 3 +%else + %define REG_SIZE 4 + %define REG_LOGAR_SIZE 2 +%endif + + %define x0 EAX + %define x1 ECX + %define x2 EDX + %define x3 EBX + %define x4 ESP + %define x5 EBP + %define x6 ESI + %define x7 EDI + + %define x0_W AX + %define x1_W CX + %define x2_W DX + %define x3_W BX + + %define x5_W BP + %define x6_W SI + %define x7_W DI + + %define x0_L AL + %define x1_L CL + %define x2_L DL + %define x3_L BL + + %define x0_H AH + %define x1_H CH + %define x2_H DH + %define x3_H BH + +; %define r0_L AL +; %define r1_L CL +; %define r2_L DL +; %define r3_L BL + +; %define r0_H AH +; %define r1_H CH +; %define r2_H DH +; %define r3_H BH + + +%if XBITS == 64 + %define x5_L BPL + %define x6_L SIL + %define x7_L DIL + %define x8_L r8b + %define x9_L r9b + %define x10_L r10b + %define x11_L r11b + %define x12_L r12b + %define x13_L r13b + %define x14_L r14b + %define x15_L r15b + + %define r0 RAX + %define r1 RCX + %define r2 RDX + %define r3 RBX + %define r4 RSP + %define r5 RBP + %define r6 RSI + %define r7 RDI + %define x8 r8d + %define x9 r9d + %define x10 r10d + %define x11 r11d + %define x12 r12d + %define x13 r13d + %define x14 r14d + %define x15 r15d +%else + %define r0 x0 + %define r1 x1 + %define r2 x2 + %define r3 x3 + %define r4 x4 + %define r5 x5 + %define r6 x6 + %define r7 x7 +%endif + + %define x0_R r0 + %define x1_R r1 + %define x2_R r2 + %define x3_R r3 + %define x4_R r4 + %define x5_R r5 + %define x6_R r6 + %define x7_R r7 + %define x8_R r8 + %define x9_R r9 + %define x10_R r10 + %define x11_R r11 + %define x12_R r12 + %define x13_R r13 + %define x14_R r14 + %define x15_R r15 + +%if (XBITS == 64) && (ABI == LINUX) + %macro MY_PUSH_2_REGS 0 + push r3 + push r5 + %endmacro + + %macro MY_POP_2_REGS 0 + pop r5 + pop r3 + %endmacro +%endif + + +%macro MY_PUSH_4_REGS 0 + push r3 + push r5 + push r6 + push r7 +%endmacro + +%macro MY_POP_4_REGS 0 + pop r7 + pop r6 + pop r5 + pop r3 +%endmacro + + +; for fastcall and for WIN-x64 +%define REG_PARAM_0_x x1 +%define REG_PARAM_0 r1 +%define REG_PARAM_1_x x2 +%define REG_PARAM_1 r2 + +%if XBITS == 32 +; for x86-fastcall + +%define REG_ABI_PARAM_0_x REG_PARAM_0_x +%define REG_ABI_PARAM_0 REG_PARAM_0 +%define REG_ABI_PARAM_1_x REG_PARAM_1_x +%define REG_ABI_PARAM_1 REG_PARAM_1 + +%macro MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 0 + MY_PUSH_4_REGS +%endmacro + +%macro MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 0 + MY_POP_4_REGS +%endmacro + +%else +; x64 + +%if ABI == WINDOWS + +; for WIN-x64: +%define REG_PARAM_2_x x8 +%define REG_PARAM_2 r8 +%define REG_PARAM_3 r9 + +%define REG_ABI_PARAM_0_x REG_PARAM_0_x +%define REG_ABI_PARAM_0 REG_PARAM_0 +%define REG_ABI_PARAM_1_x REG_PARAM_1_x +%define REG_ABI_PARAM_1 REG_PARAM_1 +%define REG_ABI_PARAM_2_x REG_PARAM_2_x +%define REG_ABI_PARAM_2 REG_PARAM_2 +%define REG_ABI_PARAM_3 REG_PARAM_3 + +%else +; for LINUX-x64: +%define REG_LINUX_PARAM_0_x x7 +%define REG_LINUX_PARAM_0 r7 +%define REG_LINUX_PARAM_1_x x6 +%define REG_LINUX_PARAM_1 r6 +%define REG_LINUX_PARAM_2 r2 +%define REG_LINUX_PARAM_3 r1 +%define REG_LINUX_PARAM_4_x x8 +%define REG_LINUX_PARAM_4 r8 +%define REG_LINUX_PARAM_5 r9 + +%define REG_ABI_PARAM_0_x REG_LINUX_PARAM_0_x +%define REG_ABI_PARAM_0 REG_LINUX_PARAM_0 +%define REG_ABI_PARAM_1_x REG_LINUX_PARAM_1_x +%define REG_ABI_PARAM_1 REG_LINUX_PARAM_1 +%define REG_ABI_PARAM_2 REG_LINUX_PARAM_2 +%define REG_ABI_PARAM_3 REG_LINUX_PARAM_3 +%define REG_ABI_PARAM_4_x REG_LINUX_PARAM_4_x +%define REG_ABI_PARAM_4 REG_LINUX_PARAM_4 +%define REG_ABI_PARAM_5 REG_LINUX_PARAM_5 + +%macro MY_ABI_LINUX_TO_WIN_2 0 + mov r2, r6 + mov r1, r7 +%endmacro + +%macro MY_ABI_LINUX_TO_WIN_3 0 + mov r8, r2 + mov r2, r6 + mov r1, r7 +%endmacro + +%macro MY_ABI_LINUX_TO_WIN_4 0 + mov r9, r1 + mov r8, r2 + mov r2, r6 + mov r1, r7 +%endmacro + +%endif ; IS_LINUX + + +%macro MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 0 + %if ABI == LINUX + MY_PUSH_2_REGS + %else + MY_PUSH_4_REGS + %endif +%endmacro + +%macro MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 0 + %if ABI == LINUX + MY_POP_2_REGS + %else + MY_POP_4_REGS + %endif +%endmacro + + +%macro MY_PUSH_PRESERVED_ABI_REGS 0 + MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 + push r12 + push r13 + push r14 + push r15 +%endmacro + + +%macro MY_POP_PRESERVED_ABI_REGS 0 + pop r15 + pop r14 + pop r13 + pop r12 + MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 +%endmacro + +%endif ; x64 + + +%macro MOVZXLOHI 3 ; dest, src, suffix + %assign n %find(%2,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9) + %if n == 0 + %fatal + %else + ; x ==> x_L or x_H + movzx %1, %tok(%strcat("x", %eval(n-1), %3)) + %endif +%endmacro + +%macro MOVZXLO 2 ; dest, src + MOVZXLOHI %1, %2, "_L" +%endmacro + +%macro MOVZXHI 2 ; dest, src + MOVZXLOHI %1, %2, "_H" +%endmacro + + +%define mod %% +%define AddNum(x, n) %tok(%strcat(x, %eval(n))) +%define XMM_REG(n) AddNum(xmm, n) +%define YMM_REG(n) AddNum(ymm, n) + + +%assign labelnum 0 +%define anonlabel "..@mylabel_" + +; automatically numbered label +%macro @@ 1 + %ifnidn %1, : + %fatal + %endif + AddNum(anonlabel, labelnum)%1 + %assign labelnum labelnum+1 +%endmacro + +%define @F AddNum(anonlabel, labelnum) ; forward to next @@ label +%define @B AddNum(anonlabel, labelnum-1) ; back to previous @@ label + diff --git a/Asm/x86/7zCrcOpt.asm b/Asm/x86/7zCrcOpt.asm index c5de80883..1f71bcec3 100644 --- a/Asm/x86/7zCrcOpt.asm +++ b/Asm/x86/7zCrcOpt.asm @@ -1,222 +1,220 @@ ; 7zCrcOpt.asm -- CRC32 calculation : optimized version ; 2023-12-08 : Igor Pavlov : Public domain -include 7zAsm.asm +%include "7zAsm.inc" MY_ASM_START -NUM_WORDS equ 3 -UNROLL_CNT equ 2 - -if (NUM_WORDS lt 1) or (NUM_WORDS gt 64) -.err -endif -if (UNROLL_CNT lt 1) -.err -endif - -rD equ r2 -rD_x equ x2 -rN equ r7 -rT equ r5 - -ifndef x64 - if (IS_CDECL gt 0) - crc_OFFS equ (REG_SIZE * 5) - data_OFFS equ (REG_SIZE + crc_OFFS) - size_OFFS equ (REG_SIZE + data_OFFS) - else - size_OFFS equ (REG_SIZE * 5) - endif - table_OFFS equ (REG_SIZE + size_OFFS) -endif +%define NUM_WORDS 3 +%define UNROLL_CNT 2 + +%if (NUM_WORDS < 1) || (NUM_WORDS > 64) + %fatal +%endif +%if UNROLL_CNT < 1 + %fatal +%endif + +%define rD r2 +%define rD_x x2 +%define rN r7 +%define rT r5 + +%if XBITS == 32 + %if IS_CDECL == 1 + %define crc_OFFS (REG_SIZE * 5) + %define data_OFFS (REG_SIZE + crc_OFFS) + %define size_OFFS (REG_SIZE + data_OFFS) + %else + %define size_OFFS (REG_SIZE * 5) + %endif + %define table_OFFS (REG_SIZE + size_OFFS) +%endif ; rN + rD is same speed as rD, but we reduce one instruction in loop -SRCDAT_1 equ rN + rD * 1 + 1 * -SRCDAT_4 equ rN + rD * 1 + 4 * +%define SRCDAT_1 rN + rD * 1 + 1 * +%define SRCDAT_4 rN + rD * 1 + 4 * -CRC macro op:req, dest:req, src:req, t:req - op dest, dword ptr [rT + @CatStr(src, _R) * 4 + 0400h * (t)] -endm -CRC_XOR macro dest:req, src:req, t:req - CRC xor, dest, src, t -endm +%macro CRC 4 ; op, dest, src, t + %assign n %find(%3, x0,x1,x2,x3,x4,x5,x6,x7,x8,x9) + %if n == 0 + %fatal + %else + ; x ==> r + %1 %2, [rT + %tok(%strcat('r', %eval(n-1))) * 4 + 0400h * (%4)] + %endif +%endmacro -CRC_MOV macro dest:req, src:req, t:req - CRC mov, dest, src, t -endm +%macro CRC_XOR 3 ; dest, src, t + CRC xor, %1, %2, %3 +%endmacro -MOVZXLO macro dest:req, src:req - movzx dest, @CatStr(src, _L) -endm - -MOVZXHI macro dest:req, src:req - movzx dest, @CatStr(src, _H) -endm +%macro CRC_MOV 3 ; dest, src, t + CRC mov, %1, %2, %3 +%endmacro ; movzx x0, x0_L - is slow in some cpus (ivb), if same register for src and dest ; movzx x3, x0_L sometimes is 0 cycles latency (not always) ; movzx x3, x0_L sometimes is 0.5 cycles latency ; movzx x3, x0_H is 2 cycles latency in some cpus -CRC1b macro - movzx x6, byte ptr [rD] +%macro CRC1b 0 + movzx x6, byte [rD] MOVZXLO x3, x0 inc rD shr x0, 8 xor x6, x3 CRC_XOR x0, x6, 0 dec rN -endm +%endmacro -LOAD_1 macro dest:req, t:req, iter:req, index:req - movzx dest, byte ptr [SRCDAT_1 (4 * (NUM_WORDS - 1 - t + iter * NUM_WORDS) + index)] -endm +%macro LOAD_1 4 ; dest, t, iter, index + movzx %1, byte [SRCDAT_1 (4 * (NUM_WORDS - 1 - %2 + %3 * NUM_WORDS) + %4)] +%endmacro -LOAD_2 macro dest:req, t:req, iter:req, index:req - movzx dest, word ptr [SRCDAT_1 (4 * (NUM_WORDS - 1 - t + iter * NUM_WORDS) + index)] -endm +%macro LOAD_2 4 ; dest, t, iter, index + movzx %1, word [SRCDAT_1 (4 * (NUM_WORDS - 1 - %2 + %3 * NUM_WORDS) + %4)] +%endmacro -CRC_QUAD macro nn, t:req, iter:req -ifdef x64 +%macro CRC_QUAD 3 ; nn, t, iter + %if XBITS == 64 ; paired memory loads give 1-3% speed gain, but it uses more registers - LOAD_2 x3, t, iter, 0 - LOAD_2 x9, t, iter, 2 + LOAD_2 x3, %2, %3, 0 + LOAD_2 x9, %2, %3, 2 MOVZXLO x6, x3 shr x3, 8 - CRC_XOR nn, x6, t * 4 + 3 + CRC_XOR %1, x6, %2 * 4 + 3 MOVZXLO x6, x9 shr x9, 8 - CRC_XOR nn, x3, t * 4 + 2 - CRC_XOR nn, x6, t * 4 + 1 - CRC_XOR nn, x9, t * 4 + 0 -elseif 0 - LOAD_2 x3, t, iter, 0 + CRC_XOR %1, x3, %2 * 4 + 2 + CRC_XOR %1, x6, %2 * 4 + 1 + CRC_XOR %1, x9, %2 * 4 + 0 + %elif 0 + LOAD_2 x3, %2, %3, 0 MOVZXLO x6, x3 shr x3, 8 - CRC_XOR nn, x6, t * 4 + 3 - CRC_XOR nn, x3, t * 4 + 2 - LOAD_2 x3, t, iter, 2 + CRC_XOR %1, x6, %2 * 4 + 3 + CRC_XOR %1, x3, %2 * 4 + 2 + LOAD_2 x3, %2, %3, 2 MOVZXLO x6, x3 shr x3, 8 - CRC_XOR nn, x6, t * 4 + 1 - CRC_XOR nn, x3, t * 4 + 0 -elseif 0 - LOAD_1 x3, t, iter, 0 - LOAD_1 x6, t, iter, 1 - CRC_XOR nn, x3, t * 4 + 3 - CRC_XOR nn, x6, t * 4 + 2 - LOAD_1 x3, t, iter, 2 - LOAD_1 x6, t, iter, 3 - CRC_XOR nn, x3, t * 4 + 1 - CRC_XOR nn, x6, t * 4 + 0 -else + CRC_XOR %1, x6, %2 * 4 + 1 + CRC_XOR %1, x3, %2 * 4 + 0 + %elif 0 + LOAD_1 x3, %2, %3, 0 + LOAD_1 x6, %2, %3, 1 + CRC_XOR %1, x3, %2 * 4 + 3 + CRC_XOR %1, x6, %2 * 4 + 2 + LOAD_1 x3, %2, %3, 2 + LOAD_1 x6, %2, %3, 3 + CRC_XOR %1, x3, %2 * 4 + 1 + CRC_XOR %1, x6, %2 * 4 + 0 + %else ; 32-bit load is better if there is only one read port (core2) ; but that code can be slower if there are 2 read ports (snb) - mov x3, dword ptr [SRCDAT_1 (4 * (NUM_WORDS - 1 - t + iter * NUM_WORDS) + 0)] + mov x3, dword [SRCDAT_1 (4 * (NUM_WORDS - 1 - %2 + %3 * NUM_WORDS) + 0)] MOVZXLO x6, x3 - CRC_XOR nn, x6, t * 4 + 3 + CRC_XOR %1, x6, %2 * 4 + 3 MOVZXHI x6, x3 shr x3, 16 - CRC_XOR nn, x6, t * 4 + 2 + CRC_XOR %1, x6, %2 * 4 + 2 MOVZXLO x6, x3 shr x3, 8 - CRC_XOR nn, x6, t * 4 + 1 - CRC_XOR nn, x3, t * 4 + 0 -endif -endm + CRC_XOR %1, x6, %2 * 4 + 1 + CRC_XOR %1, x3, %2 * 4 + 0 + %endif +%endmacro -LAST equ (4 * (NUM_WORDS - 1)) +%define LAST (4 * (NUM_WORDS - 1)) -CRC_ITER macro qq, nn, iter - mov nn, [SRCDAT_4 (NUM_WORDS * (1 + iter))] +%macro CRC_ITER 3 ; qq, nn, iter + mov %2, [SRCDAT_4 (NUM_WORDS * (1 + %3))] - i = 0 - rept NUM_WORDS - 1 - CRC_QUAD nn, i, iter - i = i + 1 - endm + %assign i 0 + %rep NUM_WORDS - 1 + CRC_QUAD %2, i, %3 + %assign i i+1 + %endrep - MOVZXLO x6, qq - mov x3, qq + MOVZXLO x6, %1 + mov x3, %1 shr x3, 24 - CRC_XOR nn, x6, LAST + 3 - CRC_XOR nn, x3, LAST + 0 - ror qq, 16 - MOVZXLO x6, qq - shr qq, 24 - CRC_XOR nn, x6, LAST + 1 -if ((UNROLL_CNT and 1) eq 1) and (iter eq (UNROLL_CNT - 1)) - CRC_MOV qq, qq, LAST + 2 - xor qq, nn -else - CRC_XOR nn, qq, LAST + 2 -endif -endm + CRC_XOR %2, x6, LAST + 3 + CRC_XOR %2, x3, LAST + 0 + ror %1, 16 + MOVZXLO x6, %1 + shr %1, 24 + CRC_XOR %2, x6, LAST + 1 + %if ((UNROLL_CNT & 1) == 1) && (%3 == (UNROLL_CNT - 1)) + CRC_MOV %1, %1, LAST + 2 + xor %1, %2 + %else + CRC_XOR %2, %1, LAST + 2 + %endif +%endmacro ; + 4 for prefetching next 4-bytes after current iteration -NUM_BYTES_LIMIT equ (NUM_WORDS * 4 * UNROLL_CNT + 4) -ALIGN_MASK equ 3 +%define NUM_BYTES_LIMIT (NUM_WORDS * 4 * UNROLL_CNT + 4) +%define ALIGN_MASK 3 -; MY_PROC @CatStr(CrcUpdateT, 12), 4 -MY_PROC @CatStr(CrcUpdateT, %(NUM_WORDS * 4)), 4 +MY_PROC AddNum(CrcUpdateT, NUM_WORDS * 4), 4 MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 - ifdef x64 + %if XBITS == 64 mov x0, REG_ABI_PARAM_0_x ; x0 = x1(win) / x7(linux) mov rT, REG_ABI_PARAM_3 ; r5 = r9(win) / x1(linux) mov rN, REG_ABI_PARAM_2 ; r7 = r8(win) / r2(linux) ; mov rD, REG_ABI_PARAM_1 ; r2 = r2(win) - if (IS_LINUX gt 0) + %if ABI == LINUX mov rD, REG_ABI_PARAM_1 ; r2 = r6 - endif - else - if (IS_CDECL gt 0) + %endif + %else + %if IS_CDECL == 1 mov x0, [r4 + crc_OFFS] mov rD, [r4 + data_OFFS] - else + %else mov x0, REG_ABI_PARAM_0_x - endif + %endif mov rN, [r4 + size_OFFS] mov rT, [r4 + table_OFFS] - endif - + %endif + cmp rN, NUM_BYTES_LIMIT + ALIGN_MASK - jb crc_end + jb .crc_end @@: test rD_x, ALIGN_MASK ; test rD, ALIGN_MASK jz @F CRC1b jmp @B @@: - xor x0, dword ptr [rD] + xor x0, dword [rD] lea rN, [rD + rN * 1 - (NUM_BYTES_LIMIT - 1)] sub rD, rN -align 16 +ALIGN 16 @@: -unr_index = 0 -while unr_index lt UNROLL_CNT - if (unr_index and 1) eq 0 +%assign unr_index 0 +%rep UNROLL_CNT + %if (unr_index & 1) == 0 CRC_ITER x0, x1, unr_index - else + %else CRC_ITER x1, x0, unr_index - endif - unr_index = unr_index + 1 -endm + %endif + %assign unr_index unr_index+1 +%endrep add rD, NUM_WORDS * 4 * UNROLL_CNT jnc @B -if 0 +%if 0 ; byte verson add rD, rN - xor x0, dword ptr [rD] + xor x0, dword [rD] add rN, NUM_BYTES_LIMIT - 1 -else +%else ; 4-byte version add rN, 4 * NUM_WORDS * UNROLL_CNT sub rD, 4 * NUM_WORDS * UNROLL_CNT @@ -232,27 +230,26 @@ else CRC_XOR x0, x6, 1 add rD, 4 -if (NUM_WORDS * UNROLL_CNT) ne 1 +%if (NUM_WORDS * UNROLL_CNT) != 1 jc @F xor x0, [SRCDAT_4 0] jmp @B @@: -endif +%endif add rD, rN add rN, 4 - 1 - -endif - + +%endif + sub rN, rD -crc_end: +.crc_end: test rN, rN - jz func_end + jz .func_end @@: CRC1b jnz @B -func_end: +.func_end: MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 -MY_ENDP + MY_ENDP -end diff --git a/Asm/x86/AesOpt.asm b/Asm/x86/AesOpt.asm index 84bf8977f..d7e28ed31 100644 --- a/Asm/x86/AesOpt.asm +++ b/Asm/x86/AesOpt.asm @@ -1,274 +1,233 @@ ; AesOpt.asm -- AES optimized code for x86 AES hardware instructions ; 2021-12-25 : Igor Pavlov : Public domain -include 7zAsm.asm - -ifdef __ASMC__ - use_vaes_256 equ 1 -else -ifdef ymm0 - use_vaes_256 equ 1 -endif -endif - - -ifdef use_vaes_256 - ECHO "++ VAES 256" -else - ECHO "-- NO VAES 256" -endif - -ifdef x64 - ECHO "x86-64" -else - ECHO "x86" -if (IS_CDECL gt 0) - ECHO "ABI : CDECL" -else - ECHO "ABI : no CDECL : FASTCALL" -endif -endif - -if (IS_LINUX gt 0) - ECHO "ABI : LINUX" -else - ECHO "ABI : WINDOWS" -endif - -MY_ASM_START - -ifndef x64 - .686 - .xmm -endif +%include "7zAsm.inc" -; MY_ALIGN EQU ALIGN(64) -MY_ALIGN EQU - -SEG_ALIGN EQU MY_ALIGN +MY_ASM_START -MY_SEG_PROC macro name:req, numParams:req - ; seg_name equ @CatStr(_TEXT$, name) - ; seg_name SEGMENT SEG_ALIGN 'CODE' - MY_PROC name, numParams -endm -MY_SEG_ENDP macro - ; seg_name ENDS -endm +%define use_vaes_256 1 -NUM_AES_KEYS_MAX equ 15 +%define NUM_AES_KEYS_MAX 15 ; the number of push operators in function PROLOG -if (IS_LINUX eq 0) or (IS_X64 eq 0) -num_regs_push equ 2 -stack_param_offset equ (REG_SIZE * (1 + num_regs_push)) -endif - -ifdef x64 - num_param equ REG_ABI_PARAM_2 -else - if (IS_CDECL gt 0) +%define num_regs_push 2 +%define stack_param_offset (REG_SIZE * (1 + num_regs_push)) + +%if XBITS == 64 + %define num_param REG_ABI_PARAM_2 +%else + %if IS_CDECL == 1 ; size_t size ; void * data ; UInt32 * aes ; ret-ip <- (r4) - aes_OFFS equ (stack_param_offset) - data_OFFS equ (REG_SIZE + aes_OFFS) - size_OFFS equ (REG_SIZE + data_OFFS) - num_param equ [r4 + size_OFFS] - else - num_param equ [r4 + stack_param_offset] - endif -endif - -keys equ REG_PARAM_0 ; r1 -rD equ REG_PARAM_1 ; r2 -rN equ r0 + %define aes_OFFS (stack_param_offset) + %define data_OFFS (REG_SIZE + aes_OFFS) + %define size_OFFS (REG_SIZE + data_OFFS) + %define num_param [r4 + size_OFFS] + %else + %define num_param [r4 + stack_param_offset] + %endif +%endif -koffs_x equ x7 -koffs_r equ r7 +%define keys REG_PARAM_0 ; r1 +%define rD REG_PARAM_1 ; r2 +%define rN r0 -ksize_x equ x6 -ksize_r equ r6 +%define koffs_x x7 +%define koffs_r r7 -keys2 equ r3 +%define ksize_x x6 +%define ksize_r r6 -state equ xmm0 -key equ xmm0 -key_ymm equ ymm0 -key_ymm_n equ 0 +%define keys2 r3 -ifdef x64 - ways = 11 -else - ways = 4 -endif +%define state xmm0 +%define key xmm0 +%define key_ymm ymm0 +%define key_ymm_n 0 -ways_start_reg equ 1 +%if XBITS == 64 + %define ways 11 +%else + %define ways 4 +%endif -iv equ @CatStr(xmm, %(ways_start_reg + ways)) -iv_ymm equ @CatStr(ymm, %(ways_start_reg + ways)) +%define ways_start_reg 1 +%define iv XMM_REG(ways_start_reg + ways) +%define iv_ymm YMM_REG(ways_start_reg + ways) -WOP macro op, op2 - i = 0 - rept ways - op @CatStr(xmm, %(ways_start_reg + i)), op2 - i = i + 1 - endm -endm -ifndef ABI_LINUX -ifdef x64 +%if (ABI == WINDOWS) && (XBITS == 64) ; we use 32 bytes of home space in stack in WIN64-x64 -NUM_HOME_MM_REGS equ (32 / 16) +%define NUM_HOME_MM_REGS (32 / 16) ; we preserve xmm registers starting from xmm6 in WIN64-x64 -MM_START_SAVE_REG equ 6 +%define MM_START_SAVE_REG 6 + +%macro SAVE_XMM 1 ; num_used_mm_regs + %assign num_save_mm_regs %1 - MM_START_SAVE_REG -SAVE_XMM macro num_used_mm_regs:req - num_save_mm_regs = num_used_mm_regs - MM_START_SAVE_REG - if num_save_mm_regs GT 0 - num_save_mm_regs2 = num_save_mm_regs - NUM_HOME_MM_REGS + %if num_save_mm_regs > 0 + %assign num_save_mm_regs2 num_save_mm_regs - NUM_HOME_MM_REGS ; RSP is (16*x + 8) after entering the function in WIN64-x64 - stack_offset = 16 * num_save_mm_regs2 + (stack_param_offset mod 16) - - i = 0 - rept num_save_mm_regs - - if i eq NUM_HOME_MM_REGS - sub r4, stack_offset - endif - - if i lt NUM_HOME_MM_REGS - movdqa [r4 + stack_param_offset + i * 16], @CatStr(xmm, %(MM_START_SAVE_REG + i)) - else - movdqa [r4 + (i - NUM_HOME_MM_REGS) * 16], @CatStr(xmm, %(MM_START_SAVE_REG + i)) - endif - - i = i + 1 - endm - endif -endm - -RESTORE_XMM macro num_used_mm_regs:req - if num_save_mm_regs GT 0 - i = 0 - if num_save_mm_regs2 GT 0 - rept num_save_mm_regs2 - movdqa @CatStr(xmm, %(MM_START_SAVE_REG + NUM_HOME_MM_REGS + i)), [r4 + i * 16] - i = i + 1 - endm + %assign stack_offset 16 * num_save_mm_regs2 + (stack_param_offset mod 16) + + %assign i 0 + %rep num_save_mm_regs + %if i == NUM_HOME_MM_REGS + sub r4, stack_offset + %endif + + %if i < NUM_HOME_MM_REGS + movdqa [r4 + stack_param_offset + i * 16], XMM_REG(MM_START_SAVE_REG + i) + %else + movdqa [r4 + (i - NUM_HOME_MM_REGS) * 16], XMM_REG(MM_START_SAVE_REG + i) + %endif + + %assign i i+1 + %endrep + %endif +%endmacro + +%macro RESTORE_XMM 1 ; num_used_mm_regs + %assign num_save_mm_regs %1 - MM_START_SAVE_REG + + %if num_save_mm_regs > 0 + %assign num_save_mm_regs2 num_save_mm_regs - NUM_HOME_MM_REGS + %assign stack_offset 16 * num_save_mm_regs2 + (stack_param_offset mod 16) + + %assign i 0 + %if num_save_mm_regs2 > 0 + %rep num_save_mm_regs2 + movdqa XMM_REG(MM_START_SAVE_REG + NUM_HOME_MM_REGS + i), [r4 + i * 16] + %assign i i+1 + %endrep add r4, stack_offset - endif + %endif - num_low_regs = num_save_mm_regs - i - i = 0 - rept num_low_regs - movdqa @CatStr(xmm, %(MM_START_SAVE_REG + i)), [r4 + stack_param_offset + i * 16] - i = i + 1 - endm - endif -endm + %assign num_low_regs num_save_mm_regs-i + %assign i 0 + %rep num_low_regs + movdqa XMM_REG(MM_START_SAVE_REG + i), [r4 + stack_param_offset + i * 16] + %assign i i+1 + %endrep + %endif +%endmacro -endif ; x64 -endif ; ABI_LINUX +%endif ; win64 -MY_PROLOG macro num_used_mm_regs:req +%macro MY_PROLOG 1 ; num_used_mm_regs ; num_regs_push: must be equal to the number of push operators ; push r3 ; push r5 - if (IS_LINUX eq 0) or (IS_X64 eq 0) + %if (ABI == WINDOWS) || (XBITS == 32) push r6 push r7 - endif + %endif mov rN, num_param ; don't move it; num_param can use stack pointer (r4) - if (IS_X64 eq 0) - if (IS_CDECL gt 0) + %if XBITS == 32 + %if IS_CDECL == 1 mov rD, [r4 + data_OFFS] mov keys, [r4 + aes_OFFS] - endif - elseif (IS_LINUX gt 0) + %endif + %elif ABI == LINUX MY_ABI_LINUX_TO_WIN_2 - endif + %endif - - ifndef ABI_LINUX - ifdef x64 - SAVE_XMM num_used_mm_regs - endif - endif + %if (ABI == WINDOWS) && (XBITS == 64) + SAVE_XMM %1 + %endif mov ksize_x, [keys + 16] shl ksize_x, 5 -endm +%endmacro -MY_EPILOG macro - ifndef ABI_LINUX - ifdef x64 - RESTORE_XMM num_save_mm_regs - endif - endif - - if (IS_LINUX eq 0) or (IS_X64 eq 0) +%macro MY_EPILOG 1 ; num_used_mm_regs + %if (ABI == WINDOWS) && (XBITS == 64) + RESTORE_XMM %1 + %endif + + %if (ABI == WINDOWS) || (XBITS == 32) pop r7 pop r6 - endif + %endif ; pop r5 ; pop r3 MY_ENDP -endm +%endmacro + + +%macro WOP 2 ; op, op2 + %assign i 0 + %rep ways + %1 XMM_REG(ways_start_reg + i), %2 + %assign i i+1 + %endrep +%endmacro -OP_KEY macro op:req, offs:req - op state, [keys + offs] -endm +%macro OP_KEY 2 ; op, offs + %1 state, [keys + %2] +%endmacro -WOP_KEY macro op:req, offs:req - movdqa key, [keys + offs] - WOP op, key -endm +%macro WOP_KEY 2 ; op, offs + movdqa key, [keys + %2] + WOP %1, key +%endmacro ; ---------- AES-CBC Decode ---------- -XOR_WITH_DATA macro reg, _ppp_ - pxor reg, [rD + i * 16] -endm +%macro WOP_DAT 1 ; op + %assign i 0 + %rep ways + %ifidni %1, pxor + pxor XMM_REG(ways_start_reg + i), [rD + i * 16] + %else + movdqa [rD + i * 16], XMM_REG(ways_start_reg + i) + %endif + %assign i i+1 + %endrep +%endmacro -WRITE_TO_DATA macro reg, _ppp_ - movdqa [rD + i * 16], reg -endm +%macro WOP_DAT2 2 ; op, offs + %assign i %2 + %rep ways - %2 + %1 XMM_REG(ways_start_reg + i), [rD + i * 16 - %2 * 16] + %assign i i+1 + %endrep +%endmacro -; state0 equ @CatStr(xmm, %(ways_start_reg)) +; %define state0 XMM_REG(ways_start_reg))) -key0 equ @CatStr(xmm, %(ways_start_reg + ways + 1)) -key0_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 1)) +%define key0 XMM_REG(ways_start_reg + ways + 1) +%define key0_ymm YMM_REG(ways_start_reg + ways + 1) -key_last equ @CatStr(xmm, %(ways_start_reg + ways + 2)) -key_last_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 2)) -key_last_ymm_n equ (ways_start_reg + ways + 2) +%define key_last XMM_REG(ways_start_reg + ways + 2) +%define key_last_ymm YMM_REG(ways_start_reg + ways + 2) +%define key_last_ymm_n (ways_start_reg + ways + 2) -NUM_CBC_REGS equ (ways_start_reg + ways + 3) +%define NUM_CBC_REGS (ways_start_reg + ways + 3) -MY_SEG_PROC AesCbc_Decode_HW, 3 - - AesCbc_Decode_HW_start:: +MY_PROC AesCbc_Decode_HW, 3 + ..@AesCbc_Decode_HW_start: MY_PROLOG NUM_CBC_REGS - - AesCbc_Decode_HW_start_2:: + + ..@AesCbc_Decode_HW_start_2: movdqa iv, [keys] add keys, 32 @@ -276,14 +235,14 @@ MY_SEG_PROC AesCbc_Decode_HW, 3 movdqa key_last, [keys] sub ksize_x, 16 - jmp check2 - align 16 - nextBlocks2: - WOP movdqa, [rD + i * 16] + jmp .check2 + ALIGN 16 + .nextBlocks2: + WOP_DAT2 movdqa, 0 mov koffs_x, ksize_x ; WOP_KEY pxor, ksize_r + 16 WOP pxor, key0 - ; align 16 + ; ALIGN 16 @@: WOP_KEY aesdec, 1 * koffs_r sub koffs_r, 16 @@ -291,26 +250,22 @@ MY_SEG_PROC AesCbc_Decode_HW, 3 ; WOP_KEY aesdeclast, 0 WOP aesdeclast, key_last - pxor @CatStr(xmm, %(ways_start_reg)), iv - i = 1 - rept ways - 1 - pxor @CatStr(xmm, %(ways_start_reg + i)), [rD + i * 16 - 16] - i = i + 1 - endm + pxor XMM_REG(ways_start_reg), iv + WOP_DAT2 pxor, 1 movdqa iv, [rD + ways * 16 - 16] - WOP WRITE_TO_DATA + WOP_DAT movdqa add rD, ways * 16 - AesCbc_Decode_HW_start_3:: - check2: + ..@AesCbc_Decode_HW_start_3: + .check2: sub rN, ways - jnc nextBlocks2 + jnc .nextBlocks2 add rN, ways sub ksize_x, 16 - jmp check - nextBlock: + jmp .check + .nextBlock: movdqa state, [rD] mov koffs_x, ksize_x ; OP_KEY pxor, 1 * ksize_r + 32 @@ -333,12 +288,13 @@ MY_SEG_PROC AesCbc_Decode_HW, 3 movdqa [rD], state add rD, 16 - check: + .check: sub rN, 1 - jnc nextBlock + jnc .nextBlock movdqa [keys - 32], iv -MY_EPILOG + + MY_EPILOG NUM_CBC_REGS @@ -346,123 +302,63 @@ MY_EPILOG ; ---------- AVX ---------- -AVX__WOP_n macro op - i = 0 - rept ways - op (ways_start_reg + i) - i = i + 1 - endm -endm - -AVX__WOP macro op - i = 0 - rept ways - op @CatStr(ymm, %(ways_start_reg + i)) - i = i + 1 - endm -endm - - -AVX__WOP_KEY macro op:req, offs:req - vmovdqa key_ymm, ymmword ptr [keys2 + offs] - AVX__WOP_n op -endm - - -AVX__CBC_START macro reg - ; vpxor reg, key_ymm, ymmword ptr [rD + 32 * i] - vpxor reg, key0_ymm, ymmword ptr [rD + 32 * i] -endm - -AVX__CBC_END macro reg - if i eq 0 - vpxor reg, reg, iv_ymm - else - vpxor reg, reg, ymmword ptr [rD + i * 32 - 16] - endif -endm - - -AVX__WRITE_TO_DATA macro reg - vmovdqu ymmword ptr [rD + 32 * i], reg -endm - -AVX__XOR_WITH_DATA macro reg - vpxor reg, reg, ymmword ptr [rD + 32 * i] -endm - -AVX__CTR_START macro reg - vpaddq iv_ymm, iv_ymm, one_ymm - ; vpxor reg, iv_ymm, key_ymm - vpxor reg, iv_ymm, key0_ymm -endm - - -MY_VAES_INSTR_2 macro cmd, dest, a1, a2 - db 0c4H - db 2 + 040H + 020h * (1 - (a2) / 8) + 080h * (1 - (dest) / 8) - db 5 + 8 * ((not (a1)) and 15) - db cmd - db 0c0H + 8 * ((dest) and 7) + ((a2) and 7) -endm - -MY_VAES_INSTR macro cmd, dest, a - MY_VAES_INSTR_2 cmd, dest, dest, a -endm - -MY_vaesenc macro dest, a - MY_VAES_INSTR 0dcH, dest, a -endm -MY_vaesenclast macro dest, a - MY_VAES_INSTR 0ddH, dest, a -endm -MY_vaesdec macro dest, a - MY_VAES_INSTR 0deH, dest, a -endm -MY_vaesdeclast macro dest, a - MY_VAES_INSTR 0dfH, dest, a -endm - - -AVX__VAES_DEC macro reg - MY_vaesdec reg, key_ymm_n -endm - -AVX__VAES_DEC_LAST_key_last macro reg - ; MY_vaesdeclast reg, key_ymm_n - MY_vaesdeclast reg, key_last_ymm_n -endm - -AVX__VAES_ENC macro reg - MY_vaesenc reg, key_ymm_n -endm - -AVX__VAES_ENC_LAST macro reg - MY_vaesenclast reg, key_ymm_n -endm - -AVX__vinserti128_TO_HIGH macro dest, src - vinserti128 dest, dest, src, 1 -endm +%macro MY_VAES_INSTR 3 ; cmd, dest, a + %1 YMM_REG(%2), YMM_REG(%2), %3 +%endmacro + +%macro AVX__WRITE_TO_DATA 0 + %assign i 0 + %rep ways + vmovdqu yword [rD + 32 * i], YMM_REG(ways_start_reg + i) + %assign i i+1 + %endrep +%endmacro + +%macro AVX__XOR_WITH_DATA 1 + %assign i %1 + %rep ways - %1 + MY_VAES_INSTR vpxor, ways_start_reg + i, yword [rD + 32 * i - %1 * 16] + %assign i i+1 + %endrep +%endmacro + +%macro AVX__CTR_START 0 + %assign i 0 + %rep ways + vpaddq iv_ymm, iv_ymm, one_ymm + ; vpxor YMM_REG(ways_start_reg + i), iv_ymm, key_ymm + vpxor YMM_REG(ways_start_reg + i), iv_ymm, key0_ymm + %assign i i+1 + %endrep +%endmacro + +%macro MY_VAES_DEC 2 ; op, key + %assign i 0 + %rep ways + MY_VAES_INSTR %1, ways_start_reg + i, %2 + %assign i i+1 + %endrep +%endmacro + MY_PROC AesCbc_Decode_HW_256, 3 - ifdef use_vaes_256 +%ifdef use_vaes_256 MY_PROLOG NUM_CBC_REGS - - cmp rN, ways * 2 - jb AesCbc_Decode_HW_start_2 - vmovdqa iv, xmmword ptr [keys] + cmp rN, ways * 2 + jb ..@AesCbc_Decode_HW_start_2 + + vmovdqa iv, [keys] add keys, 32 - vbroadcasti128 key0_ymm, xmmword ptr [keys + 1 * ksize_r] - vbroadcasti128 key_last_ymm, xmmword ptr [keys] + vbroadcasti128 key0_ymm, [keys + 1 * ksize_r] + vbroadcasti128 key_last_ymm, [keys] sub ksize_x, 16 mov koffs_x, ksize_x add ksize_x, ksize_x - - AVX_STACK_SUB = ((NUM_AES_KEYS_MAX + 1 - 2) * 32) + + %assign AVX_STACK_SUB ((NUM_AES_KEYS_MAX + 1 - 2) * 32) push keys2 sub r4, AVX_STACK_SUB ; sub r4, 32 @@ -470,93 +366,101 @@ MY_PROC AesCbc_Decode_HW_256, 3 ; lea keys2, [r4 + 32] mov keys2, r4 and keys2, -32 - broad: - vbroadcasti128 key_ymm, xmmword ptr [keys + 1 * koffs_r] - vmovdqa ymmword ptr [keys2 + koffs_r * 2], key_ymm + + .broad: + vbroadcasti128 key_ymm, [keys + 1 * koffs_r] + vmovdqa yword [keys2 + koffs_r * 2], key_ymm sub koffs_r, 16 - ; jnc broad - jnz broad + jnz .broad sub rN, ways * 2 align 16 - avx_cbcdec_nextBlock2: + .nextBlock2: mov koffs_x, ksize_x - ; AVX__WOP_KEY AVX__CBC_START, 1 * koffs_r + 32 - AVX__WOP AVX__CBC_START + + %assign i 0 + %rep ways + vpxor YMM_REG(ways_start_reg + i), key0_ymm, yword [rD + 32 * i] + %assign i i+1 + %endrep + @@: - AVX__WOP_KEY AVX__VAES_DEC, 1 * koffs_r + vmovdqa key_ymm, yword [keys2 + koffs_r] + MY_VAES_DEC vaesdec, key_ymm sub koffs_r, 32 jnz @B - ; AVX__WOP_KEY AVX__VAES_DEC_LAST, 0 - AVX__WOP_n AVX__VAES_DEC_LAST_key_last - AVX__vinserti128_TO_HIGH iv_ymm, xmmword ptr [rD] - AVX__WOP AVX__CBC_END + MY_VAES_DEC vaesdeclast, key_last_ymm + vinserti128 iv_ymm, iv_ymm, [rD], 1 + + MY_VAES_INSTR vpxor, ways_start_reg, iv_ymm + AVX__XOR_WITH_DATA 1 + + vmovdqa iv, [rD + ways * 32 - 16] + AVX__WRITE_TO_DATA - vmovdqa iv, xmmword ptr [rD + ways * 32 - 16] - AVX__WOP AVX__WRITE_TO_DATA - add rD, ways * 32 sub rN, ways * 2 - jnc avx_cbcdec_nextBlock2 + jnc .nextBlock2 add rN, ways * 2 shr ksize_x, 1 - + ; lea r4, [r4 + 1 * ksize_r + 32] add r4, AVX_STACK_SUB pop keys2 vzeroupper - jmp AesCbc_Decode_HW_start_3 - else - jmp AesCbc_Decode_HW_start - endif -MY_ENDP -MY_SEG_ENDP + jmp ..@AesCbc_Decode_HW_start_3 +%else + jmp ..@AesCbc_Decode_HW_start +%endif + MY_ENDP + ; ---------- AES-CBC Encode ---------- -e0 equ xmm1 +%define e0 xmm1 -CENC_START_KEY equ 2 -CENC_NUM_REG_KEYS equ (3 * 2) -; last_key equ @CatStr(xmm, %(CENC_START_KEY + CENC_NUM_REG_KEYS)) +%define CENC_START_KEY 2 +%define CENC_NUM_REG_KEYS (3 * 2) +%define CENC_LAST_KEY (CENC_START_KEY + CENC_NUM_REG_KEYS + 0) +; %define last_key XMM_REG(CENC_START_KEY + CENC_NUM_REG_KEYS))) -MY_SEG_PROC AesCbc_Encode_HW, 3 - MY_PROLOG (CENC_START_KEY + CENC_NUM_REG_KEYS + 0) +MY_PROC AesCbc_Encode_HW, 3 + MY_PROLOG CENC_LAST_KEY movdqa state, [keys] add keys, 32 - - i = 0 - rept CENC_NUM_REG_KEYS - movdqa @CatStr(xmm, %(CENC_START_KEY + i)), [keys + i * 16] - i = i + 1 - endm + + %assign i 0 + %rep CENC_NUM_REG_KEYS + movdqa XMM_REG(CENC_START_KEY + i), [keys + i * 16] + %assign i i+1 + %endrep add keys, ksize_r neg ksize_r add ksize_r, (16 * CENC_NUM_REG_KEYS) ; movdqa last_key, [keys] - jmp check_e + jmp .check - align 16 - nextBlock_e: + ALIGN 16 + .nextBlock: movdqa e0, [rD] mov koffs_r, ksize_r - pxor e0, @CatStr(xmm, %(CENC_START_KEY)) + pxor e0, XMM_REG(CENC_START_KEY) pxor state, e0 - - i = 1 - rept (CENC_NUM_REG_KEYS - 1) - aesenc state, @CatStr(xmm, %(CENC_START_KEY + i)) - i = i + 1 - endm + + %assign i 1 + %rep CENC_NUM_REG_KEYS - 1 + aesenc state, XMM_REG(CENC_START_KEY + i) + %assign i i+1 + %endrep @@: OP_KEY aesenc, 1 * koffs_r @@ -568,41 +472,32 @@ MY_SEG_PROC AesCbc_Encode_HW, 3 movdqa [rD], state add rD, 16 - check_e: + .check: sub rN, 1 - jnc nextBlock_e + jnc .nextBlock ; movdqa [keys - 32], state movdqa [keys + 1 * ksize_r - (16 * CENC_NUM_REG_KEYS) - 32], state -MY_EPILOG -MY_SEG_ENDP + MY_EPILOG CENC_LAST_KEY - -; ---------- AES-CTR ---------- -ifdef x64 - ; ways = 11 -endif - -one equ @CatStr(xmm, %(ways_start_reg + ways + 1)) -one_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 1)) -key0 equ @CatStr(xmm, %(ways_start_reg + ways + 2)) -key0_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 2)) -NUM_CTR_REGS equ (ways_start_reg + ways + 3) + +; ---------- AES-CTR ---------- -INIT_CTR macro reg, _ppp_ - paddq iv, one - movdqa reg, iv -endm +%define one XMM_REG(ways_start_reg + ways + 1) +%define one_ymm YMM_REG(ways_start_reg + ways + 1) +%define key0 XMM_REG(ways_start_reg + ways + 2) +%define key0_ymm YMM_REG(ways_start_reg + ways + 2) +%define NUM_CTR_REGS (ways_start_reg + ways + 3) -MY_SEG_PROC AesCtr_Code_HW, 3 - Ctr_start:: +MY_PROC AesCtr_Code_HW, 3 + ..@Ctr_start: MY_PROLOG NUM_CTR_REGS - Ctr_start_2:: + ..@Ctr_start_2: movdqa iv, [keys] add keys, 32 movdqa key0, [keys] @@ -610,15 +505,20 @@ MY_SEG_PROC AesCtr_Code_HW, 3 add keys, ksize_r neg ksize_r add ksize_r, 16 - - Ctr_start_3:: + + ..@Ctr_start_3: mov koffs_x, 1 movd one, koffs_x - jmp check2_c + jmp .check2 - align 16 - nextBlocks2_c: - WOP INIT_CTR, 0 + ALIGN 16 + .nextBlocks2: + %assign i 0 + %rep ways + paddq iv, one + movdqa XMM_REG(ways_start_reg + i), iv + %assign i i+1 + %endrep mov koffs_r, ksize_r ; WOP_KEY pxor, 1 * koffs_r -16 WOP pxor, key0 @@ -627,28 +527,27 @@ MY_SEG_PROC AesCtr_Code_HW, 3 add koffs_r, 16 jnz @B WOP_KEY aesenclast, 0 - - WOP XOR_WITH_DATA - WOP WRITE_TO_DATA + WOP_DAT pxor + WOP_DAT movdqa add rD, ways * 16 - check2_c: + .check2: sub rN, ways - jnc nextBlocks2_c + jnc .nextBlocks2 add rN, ways sub keys, 16 add ksize_r, 16 - - jmp check_c - ; align 16 - nextBlock_c: + jmp .check + + ; ALIGN 16 + .nextBlock: paddq iv, one ; movdqa state, [keys + 1 * koffs_r - 16] movdqa state, key0 mov koffs_r, ksize_r pxor state, iv - + @@: OP_KEY aesenc, 1 * koffs_r OP_KEY aesenc, 1 * koffs_r + 16 @@ -656,87 +555,90 @@ MY_SEG_PROC AesCtr_Code_HW, 3 jnz @B OP_KEY aesenc, 0 OP_KEY aesenclast, 16 - + pxor state, [rD] movdqa [rD], state add rD, 16 - check_c: + .check: sub rN, 1 - jnc nextBlock_c + jnc .nextBlock ; movdqa [keys - 32], iv movdqa [keys + 1 * ksize_r - 16 - 32], iv -MY_EPILOG + + MY_EPILOG NUM_CTR_REGS MY_PROC AesCtr_Code_HW_256, 3 - ifdef use_vaes_256 +%ifdef use_vaes_256 MY_PROLOG NUM_CTR_REGS cmp rN, ways * 2 - jb Ctr_start_2 + jb ..@Ctr_start_2 - vbroadcasti128 iv_ymm, xmmword ptr [keys] + vbroadcasti128 iv_ymm, [keys] add keys, 32 - vbroadcasti128 key0_ymm, xmmword ptr [keys] + vbroadcasti128 key0_ymm, [keys] mov koffs_x, 1 vmovd one, koffs_x vpsubq iv_ymm, iv_ymm, one_ymm vpaddq one, one, one - AVX__vinserti128_TO_HIGH one_ymm, one - + vinserti128 one_ymm, one_ymm, one, 1 + add keys, ksize_r sub ksize_x, 16 neg ksize_r mov koffs_r, ksize_r add ksize_r, ksize_r - AVX_STACK_SUB = ((NUM_AES_KEYS_MAX + 1 - 1) * 32) + %assign AVX_STACK_SUB ((NUM_AES_KEYS_MAX + 1 - 1) * 32) push keys2 lea keys2, [r4 - 32] sub r4, AVX_STACK_SUB and keys2, -32 - vbroadcasti128 key_ymm, xmmword ptr [keys] - vmovdqa ymmword ptr [keys2], key_ymm - @@: - vbroadcasti128 key_ymm, xmmword ptr [keys + 1 * koffs_r] - vmovdqa ymmword ptr [keys2 + koffs_r * 2], key_ymm + vbroadcasti128 key_ymm, [keys] + vmovdqa yword [keys2], key_ymm + @@: + vbroadcasti128 key_ymm, [keys + 1 * koffs_r] + vmovdqa yword [keys2 + koffs_r * 2], key_ymm add koffs_r, 16 jnz @B sub rN, ways * 2 - - align 16 - avx_ctr_nextBlock2: - mov koffs_r, ksize_r - AVX__WOP AVX__CTR_START - ; AVX__WOP_KEY AVX__CTR_START, 1 * koffs_r - 32 + + ALIGN 16 + .nextBlock2: + mov koffs_r, ksize_r + AVX__CTR_START + @@: - AVX__WOP_KEY AVX__VAES_ENC, 1 * koffs_r + vmovdqa key_ymm, yword [keys2 + koffs_r] + MY_VAES_DEC vaesenc, key_ymm + add koffs_r, 32 jnz @B - AVX__WOP_KEY AVX__VAES_ENC_LAST, 0 - - AVX__WOP AVX__XOR_WITH_DATA - AVX__WOP AVX__WRITE_TO_DATA - + + vmovdqa key_ymm, yword [keys2] + MY_VAES_DEC vaesenclast, key_ymm + + AVX__XOR_WITH_DATA 0 + AVX__WRITE_TO_DATA + add rD, ways * 32 sub rN, ways * 2 - jnc avx_ctr_nextBlock2 + jnc .nextBlock2 add rN, ways * 2 - + vextracti128 iv, iv_ymm, 1 sar ksize_r, 1 - + add r4, AVX_STACK_SUB pop keys2 - + vzeroupper - jmp Ctr_start_3 - else - jmp Ctr_start - endif -MY_ENDP -MY_SEG_ENDP - -end + jmp ..@Ctr_start_3 +%else + jmp ..@Ctr_start +%endif + MY_ENDP + diff --git a/Asm/x86/LzFindOpt.asm b/Asm/x86/LzFindOpt.asm index 94c5c768c..a73194603 100644 --- a/Asm/x86/LzFindOpt.asm +++ b/Asm/x86/LzFindOpt.asm @@ -1,186 +1,176 @@ ; LzFindOpt.asm -- ASM version of GetMatchesSpecN_2() function ; 2024-06-18: Igor Pavlov : Public domain -; -ifndef x64 -; x64=1 -; .err -endif +%include "7zAsm.inc" -include 7zAsm.asm +%if XBITS != 64 + %fatal +%endif MY_ASM_START -ifndef Z7_LZ_FIND_OPT_ASM_USE_SEGMENT -if (IS_LINUX gt 0) - Z7_LZ_FIND_OPT_ASM_USE_SEGMENT equ 1 -else - Z7_LZ_FIND_OPT_ASM_USE_SEGMENT equ 1 -endif -endif - -ifdef Z7_LZ_FIND_OPT_ASM_USE_SEGMENT -_TEXT$LZFINDOPT SEGMENT ALIGN(64) 'CODE' -MY_ALIGN macro num:req - align num - ; align 16 -endm -else -MY_ALIGN macro num:req +%define Z7_LZ_FIND_OPT_ASM_USE_SEGMENT 1 + +%ifdef Z7_LZ_FIND_OPT_ASM_USE_SEGMENT +%macro MY_ALIGN 1 + ALIGN %1 +%endmacro +%else +%macro MY_ALIGN 1 ; We expect that ".text" is aligned for 16-bytes. ; So we don't need large alignment inside our function. - align 16 -endm -endif + ALIGN 16 +%endmacro +%endif -MY_ALIGN_16 macro +%macro MY_ALIGN_16 0 MY_ALIGN 16 -endm +%endmacro -MY_ALIGN_32 macro +%macro MY_ALIGN_32 0 MY_ALIGN 32 -endm +%endmacro -MY_ALIGN_64 macro +%macro MY_ALIGN_64 0 MY_ALIGN 64 -endm - - -t0_L equ x0_L -t0_x equ x0 -t0 equ r0 -t1_x equ x3 -t1 equ r3 - -cp_x equ t1_x -cp_r equ t1 -m equ x5 -m_r equ r5 -len_x equ x6 -len equ r6 -diff_x equ x7 -diff equ r7 -len0 equ r10 -len1_x equ x11 -len1 equ r11 -maxLen_x equ x12 -maxLen equ r12 -d equ r13 -ptr0 equ r14 -ptr1 equ r15 - -d_lim equ m_r -cycSize equ len_x -hash_lim equ len0 -delta1_x equ len1_x -delta1_r equ len1 -delta_x equ maxLen_x -delta_r equ maxLen -hash equ ptr0 -src equ ptr1 - - - -if (IS_LINUX gt 0) +%endmacro + + +%define t0_L x0_L +%define t0_x x0 +%define t0 r0 +%define t1_x x3 +%define t1 r3 + +%define cp_x t1_x +%define cp_r t1 +%define m x5 +%define m_r r5 +%define len_x x6 +%define len r6 +%define diff_x x7 +%define diff r7 +%define len0 r10 +%define len1_x x11 +%define len1 r11 +%define maxLen_x x12 +%define maxLen r12 +%define d r13 +%define ptr0 r14 +%define ptr1 r15 + +%define d_lim m_r +%define cycSize len_x +%define hash_lim len0 +%define delta1_x len1_x +%define delta1_r len1 +%define delta_x maxLen_x +%define delta_r maxLen +%define hash ptr0 +%define src ptr1 + + + +%if ABI == LINUX ; r1 r2 r8 r9 : win32 ; r7 r6 r2 r1 r8 r9 : linux -lenLimit equ r8 -lenLimit_x equ x8 -; pos_r equ r2 -pos equ x2 -cur equ r1 -son equ r9 - -else - -lenLimit equ REG_ABI_PARAM_2 -lenLimit_x equ REG_ABI_PARAM_2_x -pos equ REG_ABI_PARAM_1_x -cur equ REG_ABI_PARAM_0 -son equ REG_ABI_PARAM_3 - -endif - - -if (IS_LINUX gt 0) - maxLen_OFFS equ (REG_SIZE * (6 + 1)) -else - cutValue_OFFS equ (REG_SIZE * (8 + 1 + 4)) - d_OFFS equ (REG_SIZE + cutValue_OFFS) - maxLen_OFFS equ (REG_SIZE + d_OFFS) -endif - hash_OFFS equ (REG_SIZE + maxLen_OFFS) - limit_OFFS equ (REG_SIZE + hash_OFFS) - size_OFFS equ (REG_SIZE + limit_OFFS) - cycPos_OFFS equ (REG_SIZE + size_OFFS) - cycSize_OFFS equ (REG_SIZE + cycPos_OFFS) - posRes_OFFS equ (REG_SIZE + cycSize_OFFS) +%define lenLimit r8 +%define lenLimit_x x8 +; %define pos_r r2 +%define pos x2 +%define cur r1 +%define son r9 + +%else + +%define lenLimit REG_ABI_PARAM_2 +%define lenLimit_x REG_ABI_PARAM_2_x +%define pos REG_ABI_PARAM_1_x +%define cur REG_ABI_PARAM_0 +%define son REG_ABI_PARAM_3 + +%endif + + +%if ABI == LINUX + %define maxLen_OFFS (REG_SIZE * (6 + 1)) +%else + %define cutValue_OFFS (REG_SIZE * (8 + 1 + 4)) + %define d_OFFS (REG_SIZE + cutValue_OFFS) + %define maxLen_OFFS (REG_SIZE + d_OFFS) +%endif + %define hash_OFFS (REG_SIZE + maxLen_OFFS) + %define limit_OFFS (REG_SIZE + hash_OFFS) + %define size_OFFS (REG_SIZE + limit_OFFS) + %define cycPos_OFFS (REG_SIZE + size_OFFS) + %define cycSize_OFFS (REG_SIZE + cycPos_OFFS) + %define posRes_OFFS (REG_SIZE + cycSize_OFFS) -if (IS_LINUX gt 0) -else - cutValue_PAR equ [r0 + cutValue_OFFS] - d_PAR equ [r0 + d_OFFS] -endif - maxLen_PAR equ [r0 + maxLen_OFFS] - hash_PAR equ [r0 + hash_OFFS] - limit_PAR equ [r0 + limit_OFFS] - size_PAR equ [r0 + size_OFFS] - cycPos_PAR equ [r0 + cycPos_OFFS] - cycSize_PAR equ [r0 + cycSize_OFFS] - posRes_PAR equ [r0 + posRes_OFFS] - - - cutValue_VAR equ DWORD PTR [r4 + 8 * 0] - cutValueCur_VAR equ DWORD PTR [r4 + 8 * 0 + 4] - cycPos_VAR equ DWORD PTR [r4 + 8 * 1 + 0] - cycSize_VAR equ DWORD PTR [r4 + 8 * 1 + 4] - hash_VAR equ QWORD PTR [r4 + 8 * 2] - limit_VAR equ QWORD PTR [r4 + 8 * 3] - size_VAR equ QWORD PTR [r4 + 8 * 4] - distances equ QWORD PTR [r4 + 8 * 5] - maxLen_VAR equ QWORD PTR [r4 + 8 * 6] - - Old_RSP equ QWORD PTR [r4 + 8 * 7] - LOCAL_SIZE equ 8 * 8 - -COPY_VAR_32 macro dest_var, src_var - mov x3, src_var - mov dest_var, x3 -endm - -COPY_VAR_64 macro dest_var, src_var - mov r3, src_var - mov dest_var, r3 -endm - - -ifdef Z7_LZ_FIND_OPT_ASM_USE_SEGMENT -; MY_ALIGN_64 -else +%if ABI == WINDOWS + %define cutValue_PAR [r0 + cutValue_OFFS] + %define d_PAR [r0 + d_OFFS] +%endif + %define maxLen_PAR [r0 + maxLen_OFFS] + %define hash_PAR [r0 + hash_OFFS] + %define limit_PAR [r0 + limit_OFFS] + %define size_PAR [r0 + size_OFFS] + %define cycPos_PAR [r0 + cycPos_OFFS] + %define cycSize_PAR [r0 + cycSize_OFFS] + %define posRes_PAR [r0 + posRes_OFFS] + + + %define cutValue_VAR DWORD [r4 + 8 * 0] + %define cutValueCur_VAR DWORD [r4 + 8 * 0 + 4] + %define cycPos_VAR DWORD [r4 + 8 * 1 + 0] + %define cycSize_VAR DWORD [r4 + 8 * 1 + 4] + %define hash_VAR QWORD [r4 + 8 * 2] + %define limit_VAR QWORD [r4 + 8 * 3] + %define size_VAR QWORD [r4 + 8 * 4] + %define distances QWORD [r4 + 8 * 5] + %define maxLen_VAR QWORD [r4 + 8 * 6] + + %define Old_RSP QWORD [r4 + 8 * 7] + %define LOCAL_SIZE 8 * 8 + +%macro COPY_VAR_32 2 ; dest, src + mov x3, %2 + mov %1, x3 +%endmacro + +%macro COPY_VAR_64 2 ; dest, src + mov r3, %2 + mov %1, r3 +%endmacro + + +%ifdef Z7_LZ_FIND_OPT_ASM_USE_SEGMENT + ; MY_ALIGN_64 + [section .text$LZFINDOPT align=64 exec] +%else MY_ALIGN_16 -endif +%endif MY_PROC GetMatchesSpecN_2, 13 -MY_PUSH_PRESERVED_ABI_REGS + MY_PUSH_PRESERVED_ABI_REGS mov r0, RSP lea r3, [r0 - LOCAL_SIZE] and r3, -64 mov RSP, r3 mov Old_RSP, r0 -if (IS_LINUX gt 0) +%if ABI == LINUX mov d, REG_ABI_PARAM_5 ; r13 = r9 mov cutValue_VAR, REG_ABI_PARAM_4_x ; = r8 mov son, REG_ABI_PARAM_3 ; r9 = r1 mov r8, REG_ABI_PARAM_2 ; r8 = r2 mov pos, REG_ABI_PARAM_1_x ; r2 = x6 mov r1, REG_ABI_PARAM_0 ; r1 = r7 -else +%else COPY_VAR_32 cutValue_VAR, cutValue_PAR mov d, d_PAR -endif +%endif COPY_VAR_64 limit_VAR, limit_PAR @@ -202,62 +192,62 @@ endif sub t0, lenLimit mov maxLen_VAR, t0 - jmp main_loop + jmp .main_loop MY_ALIGN_64 -fill_empty: +.fill_empty: ; ptr0 = *ptr1 = kEmptyHashValue; - mov QWORD PTR [ptr1], 0 + mov QWORD [ptr1], 0 inc pos inc cp_x - mov DWORD PTR [d - 4], 0 + mov DWORD [d - 4], 0 cmp d, limit_VAR - jae fin + jae .fin cmp hash, hash_lim - je fin + je .fin ; MY_ALIGN_64 -main_loop: +.main_loop: ; UInt32 delta = *hash++; mov diff_x, [hash] ; delta add hash, 4 ; mov cycPos_VAR, cp_x - + inc cur add d, 4 mov m, pos sub m, diff_x; ; matchPos - + ; CLzRef *ptr1 = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2; lea ptr1, [son + 8 * cp_r] ; mov cycSize, cycSize_VAR cmp pos, cycSize - jb directMode ; if (pos < cycSize_VAR) - + jb .directMode ; if (pos < cycSize_VAR) + ; CYC MODE cmp diff_x, cycSize - jae fill_empty ; if (delta >= cycSize_VAR) - + jae .fill_empty ; if (delta >= cycSize_VAR) + xor t0_x, t0_x mov cycPos_VAR, cp_x sub cp_x, diff_x - ; jae prepare_for_tree_loop + ; jae .prepare_for_tree_loop ; add cp_x, cycSize cmovb t0_x, cycSize add cp_x, t0_x ; cp_x += (cycPos < delta ? cycSize : 0) - jmp prepare_for_tree_loop - - -directMode: + jmp .prepare_for_tree_loop + + +.directMode: cmp diff_x, pos - je fill_empty ; if (delta == pos) - jae fin_error ; if (delta >= pos) - + je .fill_empty ; if (delta == pos) + jae .fin_error ; if (delta >= pos) + mov cycPos_VAR, cp_x mov cp_x, m - -prepare_for_tree_loop: + +.prepare_for_tree_loop: mov len0, lenLimit mov hash_VAR, hash ; CLzRef *ptr0 = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2 + 1; @@ -273,7 +263,7 @@ prepare_for_tree_loop: mov cutValueCur_VAR, t0_x MY_ALIGN_32 -tree_loop: +.tree_loop: neg diff mov len, len0 cmp len1, len0 @@ -281,54 +271,54 @@ tree_loop: add diff, cur mov t0_x, [son + cp_r * 8] ; prefetch - movzx t0_x, BYTE PTR [diff + 1 * len] + movzx t0_x, BYTE [diff + 1 * len] lea cp_r, [son + cp_r * 8] cmp [cur + 1 * len], t0_L - je matched_1 - - jb left_0 + je .matched_1 + + jb .left_0 mov [ptr1], m mov m, [cp_r + 4] lea ptr1, [cp_r + 4] sub diff, cur ; FIX32 - jmp next_node + jmp .next_node MY_ALIGN_32 -left_0: +.left_0: mov [ptr0], m mov m, [cp_r] mov ptr0, cp_r sub diff, cur ; FIX32 - ; jmp next_node + ; jmp .next_node ; ------------ NEXT NODE ------------ ; MY_ALIGN_32 -next_node: +.next_node: mov cycSize, cycSize_VAR dec cutValueCur_VAR - je finish_tree - + je .finish_tree + add diff_x, pos ; prev_match = pos + diff cmp m, diff_x - jae fin_error ; if (new_match >= prev_match) - + jae .fin_error ; if (new_match >= prev_match) + mov diff_x, pos sub diff_x, m ; delta = pos - new_match cmp pos, cycSize - jae cyc_mode_2 ; if (pos >= cycSize) + jae .cyc_mode_2 ; if (pos >= cycSize) mov cp_x, m test m, m - jne tree_loop ; if (m != 0) - -finish_tree: + jne .tree_loop ; if (m != 0) + +.finish_tree: ; ptr0 = *ptr1 = kEmptyHashValue; - mov DWORD PTR [ptr0], 0 - mov DWORD PTR [ptr1], 0 + mov DWORD [ptr0], 0 + mov DWORD [ptr1], 0 inc pos - + ; _distances[-1] = (UInt32)(d - _distances); mov t0, distances mov t1, d @@ -337,75 +327,75 @@ finish_tree: mov [t0 - 4], t1_x cmp d, limit_VAR - jae fin ; if (d >= limit) - + jae .fin ; if (d >= limit) + mov cp_x, cycPos_VAR mov hash, hash_VAR mov hash_lim, size_VAR inc cp_x cmp hash, hash_lim - jne main_loop ; if (hash != size) - jmp fin - + jne .main_loop ; if (hash != size) + jmp .fin + MY_ALIGN_32 -cyc_mode_2: +.cyc_mode_2: cmp diff_x, cycSize - jae finish_tree ; if (delta >= cycSize) + jae .finish_tree ; if (delta >= cycSize) mov cp_x, cycPos_VAR xor t0_x, t0_x sub cp_x, diff_x ; cp_x = cycPos - delta cmovb t0_x, cycSize add cp_x, t0_x ; cp_x += (cycPos < delta ? cycSize : 0) - jmp tree_loop + jmp .tree_loop + - MY_ALIGN_32 -matched_1: +.matched_1: inc len ; cmp len_x, lenLimit_x - je short lenLimit_reach - movzx t0_x, BYTE PTR [diff + 1 * len] + je short .lenLimit_reach + movzx t0_x, BYTE [diff + 1 * len] cmp [cur + 1 * len], t0_L - jne mismatch + jne .mismatch + - MY_ALIGN_32 -match_loop: +.match_loop: ; while (++len != lenLimit) (len[diff] != len[0]) ; inc len ; cmp len_x, lenLimit_x - je short lenLimit_reach - movzx t0_x, BYTE PTR [diff + 1 * len] - cmp BYTE PTR [cur + 1 * len], t0_L - je match_loop + je short .lenLimit_reach + movzx t0_x, BYTE [diff + 1 * len] + cmp BYTE [cur + 1 * len], t0_L + je .match_loop -mismatch: - jb left_2 +.mismatch: + jb .left_2 mov [ptr1], m mov m, [cp_r + 4] lea ptr1, [cp_r + 4] mov len1, len - jmp max_update - + jmp .max_update + MY_ALIGN_32 -left_2: +.left_2: mov [ptr0], m mov m, [cp_r] mov ptr0, cp_r mov len0, len -max_update: +.max_update: sub diff, cur ; restore diff cmp maxLen, len - jae next_node - + jae .next_node + mov maxLen, len add len, lenLimit mov [d], len_x @@ -413,13 +403,12 @@ max_update: not t0_x mov [d + 4], t0_x add d, 8 - - jmp next_node + + jmp .next_node - MY_ALIGN_32 -lenLimit_reach: +.lenLimit_reach: mov delta_r, cur sub delta_r, diff @@ -453,17 +442,17 @@ lenLimit_reach: ; if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) ; break; cmp hash, hash_lim - je fin + je .fin cmp d, d_lim - jae fin + jae .fin cmp delta_x, [hash] - jne main_loop - movzx t0_x, BYTE PTR [diff] + jne .main_loop + movzx t0_x, BYTE [diff] cmp [cur], t0_L - jne main_loop + jne .main_loop + + ; jmp .main_loop ; bypass for debug - ; jmp main_loop ; bypass for debug - mov cycPos_VAR, cp_x shl len, 3 ; cycSize * 8 sub diff, cur ; restore diff @@ -475,13 +464,13 @@ lenLimit_reach: add src, t0 add len, son ; len = son + cycSize * 8 - + MY_ALIGN_32 -long_loop: +.long_loop: add hash, 4 - + ; *(UInt64 *)(void *)ptr = ((const UInt64 *)(const void *)ptr)[diff]; - + mov t0, [src] add src, 8 mov [cp_r], t0 @@ -489,7 +478,7 @@ long_loop: cmp src, len cmove src, son ; if end of (son) buffer is reached, we wrap to begin - mov DWORD PTR [d], 2 + mov DWORD [d], 2 mov [d + 4], lenLimit_x mov [d + 8], delta1_x add d, 12 @@ -497,44 +486,38 @@ long_loop: inc cur cmp hash, hash_lim - je long_footer + je .long_footer cmp delta_x, [hash] - jne long_footer - movzx t0_x, BYTE PTR [diff + 1 * cur] + jne .long_footer + movzx t0_x, BYTE [diff + 1 * cur] cmp [cur], t0_L - jne long_footer + jne .long_footer cmp d, d_lim - jb long_loop + jb .long_loop -long_footer: +.long_footer: sub cp_r, son shr cp_r, 3 add pos, cp_x sub pos, cycPos_VAR mov cycSize, cycSize_VAR - + cmp d, d_lim - jae fin + jae .fin cmp hash, hash_lim - jne main_loop - jmp fin - + jne .main_loop + jmp .fin -fin_error: +.fin_error: xor d, d - -fin: + +.fin: mov RSP, Old_RSP mov t0, [r4 + posRes_OFFS] mov [t0], pos mov r0, d -MY_POP_PRESERVED_ABI_REGS -MY_ENDP - -ifdef Z7_LZ_FIND_OPT_ASM_USE_SEGMENT -_TEXT$LZFINDOPT ENDS -endif + MY_POP_PRESERVED_ABI_REGS + MY_ENDP -end diff --git a/Asm/x86/LzmaDecOpt.asm b/Asm/x86/LzmaDecOpt.asm index 7c568df1d..35b58f7c9 100644 --- a/Asm/x86/LzmaDecOpt.asm +++ b/Asm/x86/LzmaDecOpt.asm @@ -8,12 +8,11 @@ ; CLzmaDec structure, (probs) array layout, input and output of ; LzmaDec_DecodeReal_*() must be equal in both versions (C / ASM). -ifndef x64 -; x64=1 -; .err -endif +%include "7zAsm.inc" -include 7zAsm.asm +%if XBITS != 64 + %fatal +%endif MY_ASM_START @@ -30,69 +29,69 @@ MY_ASM_START ; Note that Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT adds an extra section to the ELF file. ; If you don't want to get that extra section, do not define Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT. -ifndef Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT -if (IS_LINUX gt 0) - Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT equ 1 -else - Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT equ 1 -endif -endif - -ifdef Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT -_TEXT$LZMADECOPT SEGMENT ALIGN(64) 'CODE' -MY_ALIGN macro num:req - align num - ; align 16 -endm -else -MY_ALIGN macro num:req +%define Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT 1 + +%ifdef Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT +%macro MY_ALIGN 1 + ALIGN %1 +%endmacro +%else +%macro MY_ALIGN 1 ; We expect that ".text" is aligned for 16-bytes. ; So we don't need large alignment inside out function. - align 16 -endm -endif + ALIGN 16 +%endmacro +%endif -MY_ALIGN_16 macro +%macro MY_ALIGN_16 0 MY_ALIGN 16 -endm +%endmacro -MY_ALIGN_32 macro +%macro MY_ALIGN_32 0 MY_ALIGN 32 -endm +%endmacro -MY_ALIGN_64 macro +%macro MY_ALIGN_64 0 MY_ALIGN 64 -endm +%endmacro + +; %define _LZMA_SIZE_OPT 1 -; _LZMA_SIZE_OPT equ 1 +; %define _LZMA_PROB32 1 -; _LZMA_PROB32 equ 1 +%ifdef _LZMA_PROB32 + %define PSHIFT 2 + %macro PLOAD 2 ; dest, mem + mov %1, dword [%2] + %endmacro + %macro PSTORE 2 ; src, mem + mov dword [%2], %1 + %endmacro +%else + %define PSHIFT 1 -ifdef _LZMA_PROB32 - PSHIFT equ 2 - PLOAD macro dest, mem - mov dest, dword ptr [mem] - endm - PSTORE macro src, mem - mov dword ptr [mem], src - endm -else - PSHIFT equ 1 - PLOAD macro dest, mem - movzx dest, word ptr [mem] - endm - PSTORE macro src, mem - mov word ptr [mem], @CatStr(src, _W) - endm -endif + %macro PLOAD 2 ; dest, mem + movzx %1, word [%2] + %endmacro -PMULT equ (1 SHL PSHIFT) -PMULT_HALF equ (1 SHL (PSHIFT - 1)) -PMULT_2 equ (1 SHL (PSHIFT + 1)) + %macro PSTORE 2 ; src, mem + %assign n %find(%1, x0,x1,x2,x3,x4,x5,x6,x7) + %if n == 0 + %fatal + %else + ; x ==> x_W + mov word [%2], %tok(%strcat('x', %eval(n-1), "_W")) + %endif + %endmacro +%endif -kMatchSpecLen_Error_Data equ (1 SHL 9) +%define PMULT (1 << PSHIFT) +%define PMULT_HALF (1 << (PSHIFT - 1)) +%define PMULT_2 (1 << (PSHIFT + 1)) + +%define kMatchSpecLen_Error_Data (1 << 9) ; x0 range ; x1 pbPos / (prob) TREE @@ -112,220 +111,220 @@ kMatchSpecLen_Error_Data equ (1 SHL 9) ; r15 buf -cod equ x5 -cod_L equ x5_L -range equ x0 -state equ x8 -state_R equ r8 -buf equ r15 -processedPos equ x13 -kBitModelTotal_reg equ x10 +%define cod x5 +%define cod_L x5_L +%define range x0 +%define state x8 +%define state_R r8 +%define buf r15 +%define processedPos x13 +%define kBitModelTotal_reg x10 -probBranch equ x2 -probBranch_R equ r2 -probBranch_W equ x2_W +%define probBranch x2 +%define probBranch_R r2 +%define probBranch_W x2_W -pbPos equ x1 -pbPos_R equ r1 +%define pbPos x1 +%define pbPos_R r1 -cnt equ x2 -cnt_R equ r2 +%define cnt x2 +%define cnt_R r2 -lpMask_reg equ x9 -dicPos equ r14 +%define lpMask_reg x9 +%define dicPos r14 -sym equ x3 -sym_R equ r3 -sym_L equ x3_L +%define sym x3 +%define sym_R r3 +%define sym_L x3_L -probs equ r11 -dic equ r12 +%define probs r11 +%define dic r12 -t0 equ x7 -t0_W equ x7_W -t0_R equ r7 +%define t0 x7 +%define t0_R r7 +%define t0_W x7_W -prob2 equ t0 -prob2_W equ t0_W +%define prob2 t0 +%define prob2_W t0_W -t1 equ x6 -t1_R equ r6 +%define t1 x6 +%define t1_R r6 -probs_state equ t1 -probs_state_R equ t1_R +%define probs_state t1 +%define probs_state_R t1_R -prm equ r2 -match equ x9 -match_R equ r9 -offs equ x12 -offs_R equ r12 -bit equ x14 -bit_R equ r14 +%define prm r2 +%define match x9 +%define match_R r9 +%define offs x12 +%define offs_R r12 +%define bit x14 +%define bit_R r14 -sym2 equ x9 -sym2_R equ r9 +%define sym2 x9 +%define sym2_R r9 -len_temp equ x12 +%define len_temp x12 -dist equ sym -dist2 equ x9 +%define dist sym +%define dist2 x9 -kNumBitModelTotalBits equ 11 -kBitModelTotal equ (1 SHL kNumBitModelTotalBits) -kNumMoveBits equ 5 -kBitModelOffset equ ((1 SHL kNumMoveBits) - 1) -kTopValue equ (1 SHL 24) +%define kNumBitModelTotalBits 11 +%define kBitModelTotal (1 << kNumBitModelTotalBits) +%define kNumMoveBits 5 +%define kBitModelOffset ((1 << kNumMoveBits) - 1) +%define kTopValue (1 << 24) -NORM_2 macro - ; movzx t0, BYTE PTR [buf] +%macro NORM_2 0 + ; movzx t0, BYTE [buf] shl cod, 8 - mov cod_L, BYTE PTR [buf] + mov cod_L, BYTE [buf] shl range, 8 ; or cod, t0 inc buf -endm +%endmacro -NORM macro +%macro NORM 0 cmp range, kTopValue jae SHORT @F NORM_2 @@: -endm +%endmacro ; ---------- Branch MACROS ---------- -UPDATE_0 macro probsArray:req, probOffset:req, probDisp:req +%macro UPDATE_0 3 ; probsArray, probOffset, probDisp mov prob2, kBitModelTotal_reg sub prob2, probBranch shr prob2, kNumMoveBits add probBranch, prob2 - PSTORE probBranch, probOffset * 1 + probsArray + probDisp * PMULT -endm + PSTORE probBranch, %2 * 1 + %1 + %3 * PMULT +%endmacro -UPDATE_1 macro probsArray:req, probOffset:req, probDisp:req +%macro UPDATE_1 3 ; probsArray, probOffset, probDisp sub prob2, range sub cod, range mov range, prob2 mov prob2, probBranch shr probBranch, kNumMoveBits sub prob2, probBranch - PSTORE prob2, probOffset * 1 + probsArray + probDisp * PMULT -endm + PSTORE prob2, %2 * 1 + %1 + %3 * PMULT +%endmacro -CMP_COD macro probsArray:req, probOffset:req, probDisp:req - PLOAD probBranch, probOffset * 1 + probsArray + probDisp * PMULT +%macro CMP_COD 3 ; probsArray, probOffset, probDisp + PLOAD probBranch, %2 * 1 + %1 + %3 * PMULT NORM mov prob2, range shr range, kNumBitModelTotalBits imul range, probBranch cmp cod, range -endm +%endmacro -IF_BIT_1_NOUP macro probsArray:req, probOffset:req, probDisp:req, toLabel:req - CMP_COD probsArray, probOffset, probDisp - jae toLabel -endm +%macro IF_BIT_1_NOUP 4 ; probsArray, probOffset, probDisp, toLabel + CMP_COD %1, %2, %3 + jae %4 +%endmacro -IF_BIT_1 macro probsArray:req, probOffset:req, probDisp:req, toLabel:req - IF_BIT_1_NOUP probsArray, probOffset, probDisp, toLabel - UPDATE_0 probsArray, probOffset, probDisp -endm +%macro IF_BIT_1 4 ; probsArray, probOffset, probDisp, toLabel + IF_BIT_1_NOUP %1, %2, %3, %4 + UPDATE_0 %1, %2, %3 +%endmacro -IF_BIT_0_NOUP macro probsArray:req, probOffset:req, probDisp:req, toLabel:req - CMP_COD probsArray, probOffset, probDisp - jb toLabel -endm +%macro IF_BIT_0_NOUP 4 ; probsArray, probOffset, probDisp, toLabel + CMP_COD %1, %2, %3 + jb %4 +%endmacro ; ---------- CMOV MACROS ---------- -NORM_CALC macro prob:req +%macro NORM_CALC 1 ; prob NORM mov t0, range shr range, kNumBitModelTotalBits - imul range, prob + imul range, %1 sub t0, range mov t1, cod sub cod, range -endm +%endmacro -PUP macro prob:req, probPtr:req - sub t0, prob - ; only sar works for both 16/32 bit prob modes +%macro PUP 2 ; prob, probPtr + sub t0, %1 + ; only sar works for both 16/32 bit prob modes sar t0, kNumMoveBits - add t0, prob - PSTORE t0, probPtr -endm + add t0, %1 + PSTORE t0, %2 +%endmacro -PUP_SUB macro prob:req, probPtr:req, symSub:req - sbb sym, symSub - PUP prob, probPtr -endm +%macro PUP_SUB 3 ; prob, probPtr, symSub + sbb sym, %3 + PUP %1, %2 +%endmacro -PUP_COD macro prob:req, probPtr:req, symSub:req +%macro PUP_COD 3 ; prob, probPtr, symSub mov t0, kBitModelOffset cmovb cod, t1 mov t1, sym cmovb t0, kBitModelTotal_reg - PUP_SUB prob, probPtr, symSub -endm + PUP_SUB %1, %2, %3 +%endmacro -BIT_0 macro prob:req, probNext:req - PLOAD prob, probs + 1 * PMULT - PLOAD probNext, probs + 1 * PMULT_2 +%macro BIT_0 2 ; prob, probNext + PLOAD %1, probs + 1 * PMULT + PLOAD %2, probs + 1 * PMULT_2 - NORM_CALC prob + NORM_CALC %1 cmovae range, t0 PLOAD t0, probs + 1 * PMULT_2 + PMULT - cmovae probNext, t0 + cmovae %2, t0 mov t0, kBitModelOffset cmovb cod, t1 cmovb t0, kBitModelTotal_reg mov sym, 2 - PUP_SUB prob, probs + 1 * PMULT, 0 - 1 -endm + PUP_SUB %1, probs + 1 * PMULT, 0 - 1 +%endmacro -BIT_1 macro prob:req, probNext:req - PLOAD probNext, probs + sym_R * PMULT_2 +%macro BIT_1 2 ; prob, probNext + PLOAD %2, probs + sym_R * PMULT_2 add sym, sym - NORM_CALC prob + NORM_CALC %1 cmovae range, t0 PLOAD t0, probs + sym_R * PMULT + PMULT - cmovae probNext, t0 - PUP_COD prob, probs + t1_R * PMULT_HALF, 0 - 1 -endm + cmovae %2, t0 + PUP_COD %1, probs + t1_R * PMULT_HALF, 0 - 1 +%endmacro -BIT_2 macro prob:req, symSub:req +%macro BIT_2 2 ; prob, symSub add sym, sym - NORM_CALC prob + NORM_CALC %1 cmovae range, t0 - PUP_COD prob, probs + t1_R * PMULT_HALF, symSub -endm + PUP_COD %1, probs + t1_R * PMULT_HALF, %2 +%endmacro ; ---------- MATCHED LITERAL ---------- -LITM_0 macro +%macro LITM_0 0 mov offs, 256 * PMULT shl match, (PSHIFT + 1) mov bit, offs @@ -347,10 +346,10 @@ LITM_0 macro cmovb t0, kBitModelTotal_reg mov sym, 0 PUP_SUB x1, prm, -2-1 -endm +%endmacro -LITM macro +%macro LITM 0 and bit, offs lea prm, [probs + offs_R * 1] add prm, bit_R @@ -365,10 +364,10 @@ LITM macro mov bit, match cmovae range, t0 PUP_COD x1, prm + t1_R * PMULT_HALF, - 1 -endm +%endmacro -LITM_2 macro +%macro LITM_2 0 and bit, offs lea prm, [probs + offs_R * 1] add prm, bit_R @@ -379,72 +378,73 @@ LITM_2 macro cmovae range, t0 PUP_COD x1, prm + t1_R * PMULT_HALF, 256 - 1 -endm +%endmacro ; ---------- REVERSE BITS ---------- -REV_0 macro prob:req, probNext:req - ; PLOAD prob, probs + 1 * PMULT +%macro REV_0 2 ; prob, probNext + ; PLOAD %1, probs + 1 * PMULT ; lea sym2_R, [probs + 2 * PMULT] - ; PLOAD probNext, probs + 2 * PMULT - PLOAD probNext, sym2_R + ; PLOAD %2, probs + 2 * PMULT + PLOAD %2, sym2_R - NORM_CALC prob + NORM_CALC %1 cmovae range, t0 PLOAD t0, probs + 3 * PMULT - cmovae probNext, t0 + cmovae %2, t0 cmovb cod, t1 mov t0, kBitModelOffset cmovb t0, kBitModelTotal_reg lea t1_R, [probs + 3 * PMULT] cmovae sym2_R, t1_R - PUP prob, probs + 1 * PMULT -endm + PUP %1, probs + 1 * PMULT +%endmacro -REV_1 macro prob:req, probNext:req, step:req - add sym2_R, step * PMULT - PLOAD probNext, sym2_R +%macro REV_1 3 ; prob, probNext, step + add sym2_R, %3 * PMULT + PLOAD %2, sym2_R - NORM_CALC prob + NORM_CALC %1 cmovae range, t0 - PLOAD t0, sym2_R + step * PMULT - cmovae probNext, t0 + PLOAD t0, sym2_R + %3 * PMULT + cmovae %2, t0 cmovb cod, t1 mov t0, kBitModelOffset cmovb t0, kBitModelTotal_reg - lea t1_R, [sym2_R + step * PMULT] + lea t1_R, [sym2_R + %3 * PMULT] cmovae sym2_R, t1_R - PUP prob, t1_R - step * PMULT_2 -endm + PUP %1, t1_R - %3 * PMULT_2 +%endmacro -REV_2 macro prob:req, step:req +%macro REV_2 2 ; prob, step sub sym2_R, probs shr sym2, PSHIFT or sym, sym2 - NORM_CALC prob + NORM_CALC %1 cmovae range, t0 - lea t0, [sym - step] + ;lea t0, [sym - %2] + lea t0, [sym_R - %2] cmovb sym, t0 cmovb cod, t1 mov t0, kBitModelOffset cmovb t0, kBitModelTotal_reg - PUP prob, probs + sym2_R * PMULT -endm + PUP %1, probs + sym2_R * PMULT +%endmacro -REV_1_VAR macro prob:req - PLOAD prob, sym_R +%macro REV_1_VAR 1 ; prob + PLOAD %1, sym_R mov probs, sym_R add sym_R, sym2_R - NORM_CALC prob + NORM_CALC %1 cmovae range, t0 lea t0_R, [sym_R + 1 * sym2_R] @@ -455,420 +455,419 @@ REV_1_VAR macro prob:req ; cmovb t0, t1 cmovb t0, kBitModelTotal_reg add sym2, sym2 - PUP prob, probs -endm + PUP %1, probs +%endmacro -LIT_PROBS macro lpMaskParam:req +%macro LIT_PROBS 1 ; lpMaskParam ; prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); mov t0, processedPos shl t0, 8 add sym, t0 - and sym, lpMaskParam + and sym, %1 add probs_state_R, pbPos_R - mov x1, LOC lc2 - lea sym, dword ptr[sym_R + 2 * sym_R] + mov x1, LOC(lc2) + lea sym, dword [sym_R + 2 * sym_R] add probs, Literal * PMULT shl sym, x1_L add probs, sym_R UPDATE_0 probs_state_R, 0, IsMatch inc processedPos -endm - - - -kNumPosBitsMax equ 4 -kNumPosStatesMax equ (1 SHL kNumPosBitsMax) - -kLenNumLowBits equ 3 -kLenNumLowSymbols equ (1 SHL kLenNumLowBits) -kLenNumHighBits equ 8 -kLenNumHighSymbols equ (1 SHL kLenNumHighBits) -kNumLenProbs equ (2 * kLenNumLowSymbols * kNumPosStatesMax + kLenNumHighSymbols) - -LenLow equ 0 -LenChoice equ LenLow -LenChoice2 equ (LenLow + kLenNumLowSymbols) -LenHigh equ (LenLow + 2 * kLenNumLowSymbols * kNumPosStatesMax) - -kNumStates equ 12 -kNumStates2 equ 16 -kNumLitStates equ 7 - -kStartPosModelIndex equ 4 -kEndPosModelIndex equ 14 -kNumFullDistances equ (1 SHL (kEndPosModelIndex SHR 1)) - -kNumPosSlotBits equ 6 -kNumLenToPosStates equ 4 - -kNumAlignBits equ 4 -kAlignTableSize equ (1 SHL kNumAlignBits) - -kMatchMinLen equ 2 -kMatchSpecLenStart equ (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) - -kStartOffset equ 1664 -SpecPos equ (-kStartOffset) -IsRep0Long equ (SpecPos + kNumFullDistances) -RepLenCoder equ (IsRep0Long + (kNumStates2 SHL kNumPosBitsMax)) -LenCoder equ (RepLenCoder + kNumLenProbs) -IsMatch equ (LenCoder + kNumLenProbs) -kAlign equ (IsMatch + (kNumStates2 SHL kNumPosBitsMax)) -IsRep equ (kAlign + kAlignTableSize) -IsRepG0 equ (IsRep + kNumStates) -IsRepG1 equ (IsRepG0 + kNumStates) -IsRepG2 equ (IsRepG1 + kNumStates) -PosSlot equ (IsRepG2 + kNumStates) -Literal equ (PosSlot + (kNumLenToPosStates SHL kNumPosSlotBits)) -NUM_BASE_PROBS equ (Literal + kStartOffset) - -if kAlign ne 0 - .err -endif - -if NUM_BASE_PROBS ne 1984 - .err -endif - - -PTR_FIELD equ dq ? - -CLzmaDec_Asm struct - lc db ? - lp db ? - pb db ? - _pad_ db ? - dicSize dd ? - - probs_Spec PTR_FIELD - probs_1664 PTR_FIELD - dic_Spec PTR_FIELD - dicBufSize PTR_FIELD - dicPos_Spec PTR_FIELD - buf_Spec PTR_FIELD - - range_Spec dd ? - code_Spec dd ? - processedPos_Spec dd ? - checkDicSize dd ? - rep0 dd ? - rep1 dd ? - rep2 dd ? - rep3 dd ? - state_Spec dd ? - remainLen dd ? -CLzmaDec_Asm ends - - -CLzmaDec_Asm_Loc struct - OLD_RSP PTR_FIELD - lzmaPtr PTR_FIELD - _pad0_ PTR_FIELD - _pad1_ PTR_FIELD - _pad2_ PTR_FIELD - dicBufSize PTR_FIELD - probs_Spec PTR_FIELD - dic_Spec PTR_FIELD +%endmacro + + + +%define kNumPosBitsMax 4 +%define kNumPosStatesMax (1 << kNumPosBitsMax) + +%define kLenNumLowBits 3 +%define kLenNumLowSymbols (1 << kLenNumLowBits) +%define kLenNumHighBits 8 +%define kLenNumHighSymbols (1 << kLenNumHighBits) +%define kNumLenProbs (2 * kLenNumLowSymbols * kNumPosStatesMax + kLenNumHighSymbols) + +%define LenLow 0 +%define LenChoice LenLow +%define LenChoice2 (LenLow + kLenNumLowSymbols) +%define LenHigh (LenLow + 2 * kLenNumLowSymbols * kNumPosStatesMax) + +%define kNumStates 12 +%define kNumStates2 16 +%define kNumLitStates 7 + +%define kStartPosModelIndex 4 +%define kEndPosModelIndex 14 +%define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + +%define kNumPosSlotBits 6 +%define kNumLenToPosStates 4 + +%define kNumAlignBits 4 +%define kAlignTableSize (1 << kNumAlignBits) + +%define kMatchMinLen 2 +%define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +%define kStartOffset 1664 +%define SpecPos (-kStartOffset) +%define IsRep0Long (SpecPos + kNumFullDistances) +%define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +%define LenCoder (RepLenCoder + kNumLenProbs) +%define IsMatch (LenCoder + kNumLenProbs) +%define kAlign (IsMatch + (kNumStates2 << kNumPosBitsMax)) +%define IsRep (kAlign + kAlignTableSize) +%define IsRepG0 (IsRep + kNumStates) +%define IsRepG1 (IsRepG0 + kNumStates) +%define IsRepG2 (IsRepG1 + kNumStates) +%define PosSlot (IsRepG2 + kNumStates) +%define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +%define NUM_BASE_PROBS (Literal + kStartOffset) + +%if kAlign != 0 + %fatal +%endif + +%if NUM_BASE_PROBS != 1984 + %fatal +%endif + + +%define PTR_FIELD dq ? + +struc CLzmaDec_Asm + .lc db ? + .lp db ? + .pb db ? + ._pad_ db ? + .dicSize dd ? + + .probs_Spec PTR_FIELD + .probs_1664 PTR_FIELD + .dic_Spec PTR_FIELD + .dicBufSize PTR_FIELD + .dicPos_Spec PTR_FIELD + .buf_Spec PTR_FIELD + + .range_Spec dd ? + .code_Spec dd ? + .processedPos_Spec dd ? + .checkDicSize dd ? + .rep0 dd ? + .rep1 dd ? + .rep2 dd ? + .rep3 dd ? + .state_Spec dd ? + .remainLen dd ? +endstruc + + +struc CLzmaDec_Asm_Loc + .Old_RSP PTR_FIELD + .lzmaPtr PTR_FIELD + ._pad0_ PTR_FIELD + ._pad1_ PTR_FIELD + ._pad2_ PTR_FIELD + .dicBufSize PTR_FIELD + .probs_Spec PTR_FIELD + .dic_Spec PTR_FIELD - limit PTR_FIELD - bufLimit PTR_FIELD - lc2 dd ? - lpMask dd ? - pbMask dd ? - checkDicSize dd ? - - _pad_ dd ? - remainLen dd ? - dicPos_Spec PTR_FIELD - rep0 dd ? - rep1 dd ? - rep2 dd ? - rep3 dd ? -CLzmaDec_Asm_Loc ends + .limit PTR_FIELD + .bufLimit PTR_FIELD + .lc2 dd ? + .lpMask dd ? + .pbMask dd ? + .checkDicSize dd ? + ._pad_ dd ? + .remainLen dd ? + .dicPos_Spec PTR_FIELD + .rep0 dd ? + .rep1 dd ? + .rep2 dd ? + .rep3 dd ? +endstruc -GLOB_2 equ [sym_R].CLzmaDec_Asm. -GLOB equ [r1].CLzmaDec_Asm. -LOC_0 equ [r0].CLzmaDec_Asm_Loc. -LOC equ [RSP].CLzmaDec_Asm_Loc. +%define GLOB_2(x) [sym_R + CLzmaDec_Asm. %+ x] +%define GLOB(x) [r1 + CLzmaDec_Asm. %+ x] +%define LOC_0(x) [r0 + CLzmaDec_Asm_Loc. %+ x] +%define LOC(x) [RSP + CLzmaDec_Asm_Loc. %+ x] -COPY_VAR macro name - mov t0, GLOB_2 name - mov LOC_0 name, t0 -endm +%macro COPY_VAR 1 + mov t0, GLOB_2(%1) + mov LOC_0(%1), t0 +%endmacro -RESTORE_VAR macro name - mov t0, LOC name - mov GLOB name, t0 -endm +%macro RESTORE_VAR 1 + mov t0, LOC(%1) + mov GLOB(%1), t0 +%endmacro -IsMatchBranch_Pre macro reg +%macro IsMatchBranch_Pre 0 ; prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; - mov pbPos, LOC pbMask + mov pbPos, LOC(pbMask) and pbPos, processedPos shl pbPos, (kLenNumLowBits + 1 + PSHIFT) lea probs_state_R, [probs + 1 * state_R] -endm +%endmacro -IsMatchBranch macro reg +%macro IsMatchBranch 0 IsMatchBranch_Pre - IF_BIT_1 probs_state_R, pbPos_R, IsMatch, IsMatch_label -endm + IF_BIT_1 probs_state_R, pbPos_R, IsMatch, .IsMatch_label +%endmacro -CheckLimits macro reg - cmp buf, LOC bufLimit - jae fin_OK - cmp dicPos, LOC limit - jae fin_OK -endm +%macro CheckLimits 0 + cmp buf, LOC(bufLimit) + jae .fin_OK + cmp dicPos, LOC(limit) + jae .fin_OK +%endmacro ; RSP is (16x + 8) bytes aligned in WIN64-x64 -; LocalSize equ ((((SIZEOF CLzmaDec_Asm_Loc) + 7) / 16 * 16) + 8) +; %define LocalSize ((((SIZEOF CLzmaDec_Asm_Loc) + 7) / 16 * 16) + 8) -PARAM_lzma equ REG_ABI_PARAM_0 -PARAM_limit equ REG_ABI_PARAM_1 -PARAM_bufLimit equ REG_ABI_PARAM_2 +%define PARAM_lzma REG_ABI_PARAM_0 +%define PARAM_limit REG_ABI_PARAM_1 +%define PARAM_bufLimit REG_ABI_PARAM_2 -ifdef Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT -; MY_ALIGN_64 -else +%ifdef Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT + ; MY_ALIGN_64 + [section .text$LZMADECOPT align=64 exec] +%else MY_ALIGN_16 -endif +%endif MY_PROC LzmaDec_DecodeReal_3, 3 -MY_PUSH_PRESERVED_ABI_REGS + MY_PUSH_PRESERVED_ABI_REGS - lea r0, [RSP - (SIZEOF CLzmaDec_Asm_Loc)] + lea r0, [RSP - CLzmaDec_Asm_Loc_size] and r0, -128 mov r5, RSP mov RSP, r0 - mov LOC_0 Old_RSP, r5 - mov LOC_0 lzmaPtr, PARAM_lzma - - mov LOC_0 remainLen, 0 ; remainLen must be ZERO + mov LOC_0(Old_RSP), r5 + mov LOC_0(lzmaPtr), PARAM_lzma + + mov LOC_0(remainLen), dword 0 ; remainLen must be ZERO - mov LOC_0 bufLimit, PARAM_bufLimit + mov LOC_0(bufLimit), PARAM_bufLimit mov sym_R, PARAM_lzma ; CLzmaDec_Asm_Loc pointer for GLOB_2 - mov dic, GLOB_2 dic_Spec + mov dic, GLOB_2(dic_Spec) add PARAM_limit, dic - mov LOC_0 limit, PARAM_limit + mov LOC_0(limit), PARAM_limit - COPY_VAR(rep0) - COPY_VAR(rep1) - COPY_VAR(rep2) - COPY_VAR(rep3) - - mov dicPos, GLOB_2 dicPos_Spec + COPY_VAR rep0 + COPY_VAR rep1 + COPY_VAR rep2 + COPY_VAR rep3 + + mov dicPos, GLOB_2(dicPos_Spec) add dicPos, dic - mov LOC_0 dicPos_Spec, dicPos - mov LOC_0 dic_Spec, dic - - mov x1_L, GLOB_2 pb + mov LOC_0(dicPos_Spec), dicPos + mov LOC_0(dic_Spec), dic + + mov x1_L, GLOB_2(pb) mov t0, 1 shl t0, x1_L dec t0 - mov LOC_0 pbMask, t0 + mov LOC_0(pbMask), t0 ; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; ; unsigned lc = p->prop.lc; ; unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); - mov x1_L, GLOB_2 lc + mov x1_L, GLOB_2(lc) mov x2, 100h mov t0, x2 shr x2, x1_L ; inc x1 add x1_L, PSHIFT - mov LOC_0 lc2, x1 - mov x1_L, GLOB_2 lp + mov LOC_0(lc2), x1 + mov x1_L, GLOB_2(lp) shl t0, x1_L sub t0, x2 - mov LOC_0 lpMask, t0 + mov LOC_0(lpMask), t0 mov lpMask_reg, t0 - ; mov probs, GLOB_2 probs_Spec - ; add probs, kStartOffset SHL PSHIFT - mov probs, GLOB_2 probs_1664 - mov LOC_0 probs_Spec, probs + ; mov probs, GLOB_2(probs_Spec) + ; add probs, kStartOffset << PSHIFT + mov probs, GLOB_2(probs_1664) + mov LOC_0(probs_Spec), probs - mov t0_R, GLOB_2 dicBufSize - mov LOC_0 dicBufSize, t0_R + mov t0_R, GLOB_2(dicBufSize) + mov LOC_0(dicBufSize), t0_R - mov x1, GLOB_2 checkDicSize - mov LOC_0 checkDicSize, x1 + mov x1, GLOB_2(checkDicSize) + mov LOC_0(checkDicSize), x1 - mov processedPos, GLOB_2 processedPos_Spec + mov processedPos, GLOB_2(processedPos_Spec) - mov state, GLOB_2 state_Spec + mov state, GLOB_2(state_Spec) shl state, PSHIFT - mov buf, GLOB_2 buf_Spec - mov range, GLOB_2 range_Spec - mov cod, GLOB_2 code_Spec + mov buf, GLOB_2(buf_Spec) + mov range, GLOB_2(range_Spec) + mov cod, GLOB_2(code_Spec) mov kBitModelTotal_reg, kBitModelTotal xor sym, sym ; if (processedPos != 0 || checkDicSize != 0) or x1, processedPos - jz @f + jz @F add t0_R, dic cmp dicPos, dic cmovnz t0_R, dicPos - movzx sym, byte ptr[t0_R - 1] + movzx sym, byte [t0_R - 1] @@: IsMatchBranch_Pre cmp state, 4 * PMULT - jb lit_end + jb .lit_end cmp state, kNumLitStates * PMULT - jb lit_matched_end - jmp lz_end + jb .lit_matched_end + jmp .lz_end ; ---------- LITERAL ---------- MY_ALIGN_64 -lit_start: +.lit_start: xor state, state -lit_start_2: +.lit_start_2: LIT_PROBS lpMask_reg - ifdef _LZMA_SIZE_OPT + %ifdef _LZMA_SIZE_OPT PLOAD x1, probs + 1 * PMULT mov sym, 1 MY_ALIGN_16 -lit_loop: +.lit_loop: BIT_1 x1, x2 mov x1, x2 cmp sym, 127 - jbe lit_loop + jbe .lit_loop - else + %else BIT_0 x1, x2 + %rep 3 BIT_1 x2, x1 BIT_1 x1, x2 - BIT_1 x2, x1 - BIT_1 x1, x2 - BIT_1 x2, x1 - BIT_1 x1, x2 + %endrep - endif + %endif BIT_2 x2, 256 - 1 - ; mov dic, LOC dic_Spec - mov probs, LOC probs_Spec + ; mov dic, LOC(dic_Spec) + mov probs, LOC(probs_Spec) IsMatchBranch_Pre - mov byte ptr[dicPos], sym_L + mov byte [dicPos], sym_L inc dicPos CheckLimits -lit_end: - IF_BIT_0_NOUP probs_state_R, pbPos_R, IsMatch, lit_start +.lit_end: + IF_BIT_0_NOUP probs_state_R, pbPos_R, IsMatch, .lit_start - ; jmp IsMatch_label + ; jmp .IsMatch_label ; ---------- MATCHES ---------- ; MY_ALIGN_32 -IsMatch_label: +.IsMatch_label: UPDATE_1 probs_state_R, pbPos_R, IsMatch - IF_BIT_1 probs_state_R, 0, IsRep, IsRep_label + IF_BIT_1 probs_state_R, 0, IsRep, .IsRep_label add probs, LenCoder * PMULT add state, kNumStates * PMULT ; ---------- LEN DECODE ---------- -len_decode: +.len_decode: mov len_temp, 8 - 1 - kMatchMinLen - IF_BIT_0_NOUP probs, 0, 0, len_mid_0 + IF_BIT_0_NOUP probs, 0, 0, .len_mid_0 UPDATE_1 probs, 0, 0 - add probs, (1 SHL (kLenNumLowBits + PSHIFT)) + add probs, (1 << (kLenNumLowBits + PSHIFT)) mov len_temp, -1 - kMatchMinLen - IF_BIT_0_NOUP probs, 0, 0, len_mid_0 + IF_BIT_0_NOUP probs, 0, 0, .len_mid_0 UPDATE_1 probs, 0, 0 - add probs, LenHigh * PMULT - (1 SHL (kLenNumLowBits + PSHIFT)) + add probs, LenHigh * PMULT - (1 << (kLenNumLowBits + PSHIFT)) mov sym, 1 PLOAD x1, probs + 1 * PMULT MY_ALIGN_32 -len8_loop: +.len8_loop: BIT_1 x1, x2 mov x1, x2 cmp sym, 64 - jb len8_loop - + jb .len8_loop + mov len_temp, (kLenNumHighSymbols - kLenNumLowSymbols * 2) - 1 - kMatchMinLen - jmp short len_mid_2 ; we use short here for MASM that doesn't optimize that code as another assembler programs - + ;jmp short .len_mid_2 ; we use short here for MASM that doesn't optimize that code as another assembler programs + jmp .len_mid_2 + MY_ALIGN_32 -len_mid_0: +.len_mid_0: UPDATE_0 probs, 0, 0 add probs, pbPos_R BIT_0 x2, x1 -len_mid_2: +.len_mid_2: BIT_1 x1, x2 BIT_2 x2, len_temp - mov probs, LOC probs_Spec + mov probs, LOC(probs_Spec) cmp state, kNumStates * PMULT - jb copy_match - + jb .copy_match + ; ---------- DECODE DISTANCE ---------- ; probs + PosSlot + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); - mov t0, 3 + kMatchMinLen + mov t0, 3 + kMatchMinLen cmp sym, 3 + kMatchMinLen cmovb t0, sym - add probs, PosSlot * PMULT - (kMatchMinLen SHL (kNumPosSlotBits + PSHIFT)) + add probs, PosSlot * PMULT - (kMatchMinLen << (kNumPosSlotBits + PSHIFT)) shl t0, (kNumPosSlotBits + PSHIFT) add probs, t0_R - + ; sym = Len - ; mov LOC remainLen, sym + ; mov LOC(remainLen), sym mov len_temp, sym - ifdef _LZMA_SIZE_OPT + %ifdef _LZMA_SIZE_OPT PLOAD x1, probs + 1 * PMULT mov sym, 1 MY_ALIGN_16 -slot_loop: +.slot_loop: BIT_1 x1, x2 mov x1, x2 cmp sym, 32 - jb slot_loop - - else - + jb .slot_loop + + %else + BIT_0 x1, x2 + %rep 2 BIT_1 x2, x1 BIT_1 x1, x2 - BIT_1 x2, x1 - BIT_1 x1, x2 - - endif - + %endrep + + %endif + mov x1, sym BIT_2 x2, 64-1 and sym, 3 - mov probs, LOC probs_Spec + mov probs, LOC(probs_Spec) cmp x1, 32 + kEndPosModelIndex / 2 - jb short_dist + jb .short_dist ; unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); sub x1, (32 + 1 + kNumAlignBits) @@ -877,43 +876,42 @@ slot_loop: PLOAD x2, probs + 1 * PMULT shl sym, kNumAlignBits + 1 lea sym2_R, [probs + 2 * PMULT] - - jmp direct_norm - ; lea t1, [sym_R + (1 SHL kNumAlignBits)] + + jmp .direct_norm + ; lea t1, [sym_R + (1 << kNumAlignBits)] ; cmp range, kTopValue - ; jb direct_norm - + ; jb .direct_norm + ; ---------- DIRECT DISTANCE ---------- MY_ALIGN_32 -direct_loop: +.direct_loop: shr range, 1 mov t0, cod sub cod, range cmovs cod, t0 cmovns sym, t1 - - comment ~ - sub cod, range - mov x2, cod - sar x2, 31 - lea sym, dword ptr [r2 + sym_R * 2 + 1] - and x2, range - add cod, x2 - ~ + + ;sub cod, range + ;mov x2, cod + ;sar x2, 31 + ;lea sym, dword [r2 + sym_R * 2 + 1] + ;and x2, range + ;add cod, x2 + dec x1 - je direct_end + je .direct_end add sym, sym -direct_norm: - lea t1, [sym_R + (1 SHL kNumAlignBits)] +.direct_norm: + lea t1, [sym_R + (1 << kNumAlignBits)] cmp range, kTopValue - jae near ptr direct_loop + jae near .direct_loop ; we align for 32 here with "near ptr" command above NORM_2 - jmp direct_loop + jmp .direct_loop MY_ALIGN_32 -direct_end: +.direct_end: ; prob = + kAlign; ; distance <<= kNumAlignBits; REV_0 x2, x1 @@ -921,34 +919,34 @@ direct_end: REV_1 x2, x1, 4 REV_2 x1, 8 -decode_dist_end: +.decode_dist_end: ; if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) - mov t1, LOC rep0 - mov x1, LOC rep1 - mov x2, LOC rep2 - - mov t0, LOC checkDicSize + mov t1, LOC(rep0) + mov x1, LOC(rep1) + mov x2, LOC(rep2) + + mov t0, LOC(checkDicSize) test t0, t0 cmove t0, processedPos cmp sym, t0 - jae end_of_payload - ; jmp end_of_payload ; for debug - + jae .end_of_payload + ; jmp .end_of_payload ; for debug + ; rep3 = rep2; ; rep2 = rep1; ; rep1 = rep0; ; rep0 = distance + 1; inc sym - mov LOC rep0, sym - ; mov sym, LOC remainLen + mov LOC(rep0), sym + ; mov sym, LOC(remainLen) mov sym, len_temp - mov LOC rep1, t1 - mov LOC rep2, x1 - mov LOC rep3, x2 - + mov LOC(rep1), t1 + mov LOC(rep2), x1 + mov LOC(rep3), x2 + ; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; cmp state, (kNumStates + kNumLitStates) * PMULT mov state, kNumLitStates * PMULT @@ -957,7 +955,7 @@ decode_dist_end: ; ---------- COPY MATCH ---------- -copy_match: +.copy_match: ; len += kMatchMinLen; ; add sym, kMatchMinLen @@ -967,17 +965,17 @@ copy_match: ; p->dicPos = dicPos; ; return SZ_ERROR_DATA; ; } - mov cnt_R, LOC limit + mov cnt_R, LOC(limit) sub cnt_R, dicPos - jz fin_dicPos_LIMIT + jz .fin_dicPos_LIMIT ; curLen = ((rem < len) ? (unsigned)rem : len); cmp cnt_R, sym_R ; cmovae cnt_R, sym_R ; 64-bit cmovae cnt, sym ; 32-bit - mov dic, LOC dic_Spec - mov x1, LOC rep0 + mov dic, LOC(dic_Spec) + mov x1, LOC(rep0) mov t0_R, dicPos add dicPos, cnt_R @@ -985,19 +983,19 @@ copy_match: add processedPos, cnt ; len -= curLen; sub sym, cnt - mov LOC remainLen, sym + mov LOC(remainLen), sym sub t0_R, dic - + ; pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); sub t0_R, r1 - jae @f + jae @F - mov r1, LOC dicBufSize + mov r1, LOC(dicBufSize) add t0_R, r1 sub r1, t0_R cmp cnt_R, r1 - ja copy_match_cross + ja .copy_match_cross @@: ; if (curLen <= dicBufSize - pos) @@ -1010,13 +1008,13 @@ copy_match: ; const Byte *lim = dest + curLen; add t0_R, dic - movzx sym, byte ptr[t0_R] + movzx sym, byte [t0_R] add t0_R, cnt_R neg cnt_R ; lea r1, [dicPos - 1] -copy_common: +.copy_common: dec dicPos - ; cmp LOC rep0, 1 + ; cmp LOC(rep0), 1 ; je rep0Label ; t0_R - src_lim @@ -1025,143 +1023,138 @@ copy_common: IsMatchBranch_Pre inc cnt_R - jz copy_end + jz .copy_end MY_ALIGN_16 @@: - mov byte ptr[cnt_R * 1 + dicPos], sym_L - movzx sym, byte ptr[cnt_R * 1 + t0_R] + mov byte [cnt_R * 1 + dicPos], sym_L + movzx sym, byte [cnt_R * 1 + t0_R] inc cnt_R - jnz @b + jnz @B -copy_end: -lz_end_match: - mov byte ptr[dicPos], sym_L +.copy_end: +.lz_end_match: + mov byte [dicPos], sym_L inc dicPos - + ; IsMatchBranch_Pre CheckLimits -lz_end: - IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, IsMatch_label +.lz_end: + IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, .IsMatch_label ; ---------- LITERAL MATCHED ---------- - - LIT_PROBS LOC lpMask - + + LIT_PROBS LOC(lpMask) + ; matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; - mov x1, LOC rep0 - ; mov dic, LOC dic_Spec - mov LOC dicPos_Spec, dicPos - + mov x1, LOC(rep0) + ; mov dic, LOC(dic_Spec) + mov LOC(dicPos_Spec), dicPos + ; state -= (state < 10) ? 3 : 6; lea t0, [state_R - 6 * PMULT] sub state, 3 * PMULT cmp state, 7 * PMULT cmovae state, t0 - + sub dicPos, dic sub dicPos, r1 - jae @f - add dicPos, LOC dicBufSize + jae @F + add dicPos, LOC(dicBufSize) @@: - comment ~ - xor t0, t0 - sub dicPos, r1 - cmovb t0_R, LOC dicBufSize - ~ + ;xor t0, t0 + ;sub dicPos, r1 + ;cmovb t0_R, LOC(dicBufSize) - movzx match, byte ptr[dic + dicPos * 1] + movzx match, byte [dic + dicPos * 1] - ifdef _LZMA_SIZE_OPT + %ifdef _LZMA_SIZE_OPT mov offs, 256 * PMULT shl match, (PSHIFT + 1) mov bit, match mov sym, 1 MY_ALIGN_16 -litm_loop: +.litm_loop: LITM cmp sym, 256 - jb litm_loop + jb .litm_loop sub sym, 256 - - else - + + %else + LITM_0 + %rep 6 LITM - LITM - LITM - LITM - LITM - LITM + %endrep LITM_2 - - endif - - mov probs, LOC probs_Spec + + %endif + + mov probs, LOC(probs_Spec) IsMatchBranch_Pre - ; mov dic, LOC dic_Spec - mov dicPos, LOC dicPos_Spec - mov byte ptr[dicPos], sym_L + ; mov dic, LOC(dic_Spec) + mov dicPos, LOC(dicPos_Spec) + mov byte [dicPos], sym_L inc dicPos - + CheckLimits -lit_matched_end: - IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, IsMatch_label +.lit_matched_end: + IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, .IsMatch_label ; IsMatchBranch - mov lpMask_reg, LOC lpMask + mov lpMask_reg, LOC(lpMask) sub state, 3 * PMULT - jmp lit_start_2 - + jmp .lit_start_2 + ; ---------- REP 0 LITERAL ---------- MY_ALIGN_32 -IsRep0Short_label: +.IsRep0Short_label: UPDATE_0 probs_state_R, pbPos_R, IsRep0Long ; dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; - mov dic, LOC dic_Spec + mov dic, LOC(dic_Spec) mov t0_R, dicPos - mov probBranch, LOC rep0 + mov probBranch, LOC(rep0) sub t0_R, dic - + sub probs, RepLenCoder * PMULT - + ; state = state < kNumLitStates ? 9 : 11; or state, 1 * PMULT - + ; the caller doesn't allow (dicPos >= limit) case for REP_SHORT ; so we don't need the following (dicPos == limit) check here: - ; cmp dicPos, LOC limit + ; cmp dicPos, LOC(limit) ; jae fin_dicPos_LIMIT_REP_SHORT inc processedPos IsMatchBranch_Pre - -; xor sym, sym -; sub t0_R, probBranch_R -; cmovb sym_R, LOC dicBufSize -; add t0_R, sym_R + + ;xor sym, sym + ;sub t0_R, probBranch_R + ;cmovb sym_R, LOC(dicBufSize) + ;add t0_R, sym_R sub t0_R, probBranch_R - jae @f - add t0_R, LOC dicBufSize + jae @F + add t0_R, LOC(dicBufSize) @@: - movzx sym, byte ptr[dic + t0_R * 1] - jmp lz_end_match - - + movzx sym, byte [dic + t0_R * 1] + jmp .lz_end_match + + MY_ALIGN_32 -IsRep_label: +.IsRep_label: UPDATE_1 probs_state_R, 0, IsRep ; The (checkDicSize == 0 && processedPos == 0) case was checked before in LzmaDec.c with kBadRepCode. ; So we don't check it here. - + ; mov t0, processedPos - ; or t0, LOC checkDicSize + ; or t0, LOC(checkDicSize) ; jz fin_ERROR_2 ; state = state < kNumLitStates ? 8 : 11; @@ -1172,89 +1165,89 @@ IsRep_label: ; prob = probs + RepLenCoder; add probs, RepLenCoder * PMULT - - IF_BIT_1 probs_state_R, 0, IsRepG0, IsRepG0_label - IF_BIT_0_NOUP probs_state_R, pbPos_R, IsRep0Long, IsRep0Short_label + + IF_BIT_1 probs_state_R, 0, IsRepG0, .IsRepG0_label + IF_BIT_0_NOUP probs_state_R, pbPos_R, IsRep0Long, .IsRep0Short_label UPDATE_1 probs_state_R, pbPos_R, IsRep0Long - jmp len_decode + jmp .len_decode MY_ALIGN_32 -IsRepG0_label: +.IsRepG0_label: UPDATE_1 probs_state_R, 0, IsRepG0 - mov dist2, LOC rep0 - mov dist, LOC rep1 - mov LOC rep1, dist2 - - IF_BIT_1 probs_state_R, 0, IsRepG1, IsRepG1_label - mov LOC rep0, dist - jmp len_decode - + mov dist2, LOC(rep0) + mov dist, LOC(rep1) + mov LOC(rep1), dist2 + + IF_BIT_1 probs_state_R, 0, IsRepG1, .IsRepG1_label + mov LOC(rep0), dist + jmp .len_decode + ; MY_ALIGN_32 -IsRepG1_label: +.IsRepG1_label: UPDATE_1 probs_state_R, 0, IsRepG1 - mov dist2, LOC rep2 - mov LOC rep2, dist - - IF_BIT_1 probs_state_R, 0, IsRepG2, IsRepG2_label - mov LOC rep0, dist2 - jmp len_decode + mov dist2, LOC(rep2) + mov LOC(rep2), dist + + IF_BIT_1 probs_state_R, 0, IsRepG2, .IsRepG2_label + mov LOC(rep0), dist2 + jmp .len_decode ; MY_ALIGN_32 -IsRepG2_label: +.IsRepG2_label: UPDATE_1 probs_state_R, 0, IsRepG2 - mov dist, LOC rep3 - mov LOC rep3, dist2 - mov LOC rep0, dist - jmp len_decode + mov dist, LOC(rep3) + mov LOC(rep3), dist2 + mov LOC(rep0), dist + jmp .len_decode + - ; ---------- SPEC SHORT DISTANCE ---------- MY_ALIGN_32 -short_dist: +.short_dist: sub x1, 32 + 1 - jbe decode_dist_end + jbe .decode_dist_end or sym, 2 shl sym, x1_L lea sym_R, [probs + sym_R * PMULT + SpecPos * PMULT + 1 * PMULT] mov sym2, PMULT ; step MY_ALIGN_32 -spec_loop: +.spec_loop: REV_1_VAR x2 dec x1 - jnz spec_loop + jnz .spec_loop - mov probs, LOC probs_Spec + mov probs, LOC(probs_Spec) sub sym, sym2 sub sym, SpecPos * PMULT sub sym_R, probs shr sym, PSHIFT - - jmp decode_dist_end + + jmp .decode_dist_end ; ---------- COPY MATCH CROSS ---------- -copy_match_cross: +.copy_match_cross: ; t0_R - src pos ; r1 - len to dicBufSize ; cnt_R - total copy len mov t1_R, t0_R ; srcPos mov t0_R, dic - mov r1, LOC dicBufSize ; + mov r1, LOC(dicBufSize) neg cnt_R @@: - movzx sym, byte ptr[t1_R * 1 + t0_R] + movzx sym, byte [t1_R * 1 + t0_R] inc t1_R - mov byte ptr[cnt_R * 1 + dicPos], sym_L + mov byte [cnt_R * 1 + dicPos], sym_L inc cnt_R cmp t1_R, r1 - jne @b - - movzx sym, byte ptr[t0_R] + jne @B + + movzx sym, byte [t0_R] sub t0_R, cnt_R - jmp copy_common + jmp .copy_common @@ -1262,78 +1255,73 @@ copy_match_cross: ; fin_dicPos_LIMIT_REP_SHORT: ; mov sym, 1 -fin_dicPos_LIMIT: - mov LOC remainLen, sym - jmp fin_OK +.fin_dicPos_LIMIT: + mov LOC(remainLen), sym + jmp .fin_OK ; For more strict mode we can stop decoding with error ; mov sym, 1 - ; jmp fin + ; jmp .fin -fin_ERROR_MATCH_DIST: +.fin_ERROR_MATCH_DIST: ; rep3 = rep2; ; rep2 = rep1; ; rep1 = rep0; ; rep0 = distance + 1; - + add len_temp, kMatchSpecLen_Error_Data - mov LOC remainLen, len_temp + mov LOC(remainLen), len_temp + + mov LOC(rep0), sym + mov LOC(rep1), t1 + mov LOC(rep2), x1 + mov LOC(rep3), x2 - mov LOC rep0, sym - mov LOC rep1, t1 - mov LOC rep2, x1 - mov LOC rep3, x2 - ; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; cmp state, (kNumStates + kNumLitStates) * PMULT mov state, kNumLitStates * PMULT mov t0, (kNumLitStates + 3) * PMULT cmovae state, t0 - ; jmp fin_OK + ; jmp .fin_OK mov sym, 1 - jmp fin + jmp .fin -end_of_payload: +.end_of_payload: inc sym - jnz fin_ERROR_MATCH_DIST + jnz .fin_ERROR_MATCH_DIST - mov LOC remainLen, kMatchSpecLenStart + mov LOC(remainLen), dword kMatchSpecLenStart sub state, kNumStates * PMULT -fin_OK: +.fin_OK: xor sym, sym -fin: +.fin: NORM - mov r1, LOC lzmaPtr + mov r1, LOC(lzmaPtr) - sub dicPos, LOC dic_Spec - mov GLOB dicPos_Spec, dicPos - mov GLOB buf_Spec, buf - mov GLOB range_Spec, range - mov GLOB code_Spec, cod + sub dicPos, LOC(dic_Spec) + mov GLOB(dicPos_Spec), dicPos + mov GLOB(buf_Spec), buf + mov GLOB(range_Spec), range + mov GLOB(code_Spec), cod shr state, PSHIFT - mov GLOB state_Spec, state - mov GLOB processedPos_Spec, processedPos + mov GLOB(state_Spec), state + mov GLOB(processedPos_Spec), processedPos - RESTORE_VAR(remainLen) - RESTORE_VAR(rep0) - RESTORE_VAR(rep1) - RESTORE_VAR(rep2) - RESTORE_VAR(rep3) + RESTORE_VAR remainLen + RESTORE_VAR rep0 + RESTORE_VAR rep1 + RESTORE_VAR rep2 + RESTORE_VAR rep3 mov x0, sym - - mov RSP, LOC Old_RSP -MY_POP_PRESERVED_ABI_REGS -MY_ENDP + mov RSP, LOC(Old_RSP) -ifdef Z7_LZMA_DEC_OPT_ASM_USE_SEGMENT -_TEXT$LZMADECOPT ENDS -endif + MY_POP_PRESERVED_ABI_REGS + MY_ENDP -end diff --git a/Asm/x86/Sha1Opt.asm b/Asm/x86/Sha1Opt.asm index 0b63aeb85..372774e41 100644 --- a/Asm/x86/Sha1Opt.asm +++ b/Asm/x86/Sha1Opt.asm @@ -1,263 +1,200 @@ ; Sha1Opt.asm -- SHA-1 optimized code for SHA-1 x86 hardware instructions ; 2024-06-16 : Igor Pavlov : Public domain -include 7zAsm.asm +%include "7zAsm.inc" MY_ASM_START +%if XBITS == 64 + %define rNum REG_ABI_PARAM_2 + %if ABI == WINDOWS + %define LOCAL_SIZE (16 * 2) + %endif +%else + %define rNum r0 + %define LOCAL_SIZE (16 * 1) +%endif +%define rState REG_ABI_PARAM_0 +%define rData REG_ABI_PARAM_1 - - - - - - - - - - - -CONST SEGMENT READONLY - -align 16 -Reverse_Endian_Mask db 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0 - - - - - - - - - - - - - - - - - - - - - - -CONST ENDS - -; _TEXT$SHA1OPT SEGMENT 'CODE' - -ifndef x64 - .686 - .xmm -endif - -ifdef x64 - rNum equ REG_ABI_PARAM_2 - if (IS_LINUX eq 0) - LOCAL_SIZE equ (16 * 2) - endif -else - rNum equ r0 - LOCAL_SIZE equ (16 * 1) -endif - -rState equ REG_ABI_PARAM_0 -rData equ REG_ABI_PARAM_1 - - -MY_sha1rnds4 macro a1, a2, imm - db 0fH, 03aH, 0ccH, (0c0H + a1 * 8 + a2), imm -endm - -MY_SHA_INSTR macro cmd, a1, a2 - db 0fH, 038H, cmd, (0c0H + a1 * 8 + a2) -endm - -cmd_sha1nexte equ 0c8H -cmd_sha1msg1 equ 0c9H -cmd_sha1msg2 equ 0caH - -MY_sha1nexte macro a1, a2 - MY_SHA_INSTR cmd_sha1nexte, a1, a2 -endm - -MY_sha1msg1 macro a1, a2 - MY_SHA_INSTR cmd_sha1msg1, a1, a2 -endm - -MY_sha1msg2 macro a1, a2 - MY_SHA_INSTR cmd_sha1msg2, a1, a2 -endm - -MY_PROLOG macro - ifdef x64 - if (IS_LINUX eq 0) +%macro MY_PROLOG 0 + %if XBITS == 64 + %if ABI == WINDOWS movdqa [r4 + 8], xmm6 movdqa [r4 + 8 + 16], xmm7 sub r4, LOCAL_SIZE + 8 movdqa [r4 ], xmm8 movdqa [r4 + 16], xmm9 - endif - else ; x86 - if (IS_CDECL gt 0) - mov rState, [r4 + REG_SIZE * 1] - mov rData, [r4 + REG_SIZE * 2] - mov rNum, [r4 + REG_SIZE * 3] - else ; fastcall - mov rNum, [r4 + REG_SIZE * 1] - endif + %endif + %else ; x86 + %if IS_CDECL == 1 + mov rState, dword [r4 + REG_SIZE * 1] + mov rData, dword [r4 + REG_SIZE * 2] + mov rNum, dword [r4 + REG_SIZE * 3] + %else ; fastcall + mov rNum, dword [r4 + REG_SIZE * 1] + %endif push r5 mov r5, r4 and r4, -16 sub r4, LOCAL_SIZE - endif -endm + %endif +%endmacro -MY_EPILOG macro - ifdef x64 - if (IS_LINUX eq 0) +%macro MY_EPILOG 0 + %if XBITS == 64 + %if ABI == WINDOWS movdqa xmm8, [r4] movdqa xmm9, [r4 + 16] add r4, LOCAL_SIZE + 8 movdqa xmm6, [r4 + 8] movdqa xmm7, [r4 + 8 + 16] - endif - else ; x86 + %endif + %else ; x86 mov r4, r5 pop r5 - endif - MY_ENDP -endm + %endif + MY_ENDP +%endmacro -e0_N equ 0 -e1_N equ 1 -abcd_N equ 2 -e0_save_N equ 3 -w_regs equ 4 +%define e0_N 0 +%define e1_N 1 +%define abcd_N 2 +%define e0_save_N 3 +%define w_regs 4 -e0 equ @CatStr(xmm, %e0_N) -e1 equ @CatStr(xmm, %e1_N) -abcd equ @CatStr(xmm, %abcd_N) -e0_save equ @CatStr(xmm, %e0_save_N) +%define e0 XMM_REG(e0_N) +%define e1 XMM_REG(e1_N) +%define abcd XMM_REG(abcd_N) +%define e0_save XMM_REG(e0_save_N) -ifdef x64 - abcd_save equ xmm8 - mask2 equ xmm9 -else - abcd_save equ [r4] - mask2 equ e1 -endif +%if XBITS == 64 + %define abcd_save xmm8 + %define mask2 xmm9 +%else + %define abcd_save [r4] + %define mask2 e1 +%endif -LOAD_MASK macro - movdqa mask2, XMMWORD PTR Reverse_Endian_Mask -endm +%macro LOAD_MASK 0 + movdqa mask2, [.Reverse_Endian_Mask] +%endmacro -LOAD_W macro k:req - movdqu @CatStr(xmm, %(w_regs + k)), [rData + (16 * (k))] - pshufb @CatStr(xmm, %(w_regs + k)), mask2 -endm +%macro LOAD_W 1 + movdqu XMM_REG(w_regs + %1), [rData + 16 * %1] + pshufb XMM_REG(w_regs + %1), mask2 +%endmacro ; pre2 can be 2 or 3 (recommended) -pre2 equ 3 -pre1 equ (pre2 + 1) +%define pre2 3 +%define pre1 (pre2 + 1) + +%define NUM_ROUNDS4 20 + -NUM_ROUNDS4 equ 20 - -RND4 macro k - movdqa @CatStr(xmm, %(e0_N + ((k + 1) mod 2))), abcd - MY_sha1rnds4 abcd_N, (e0_N + (k mod 2)), k / 5 +%macro MY_XMMCMD 3 ; op, reg1, reg2 + %1 XMM_REG(%2), XMM_REG(%3) +%endmacro - nextM = (w_regs + ((k + 1) mod 4)) +%macro RND4 1 + MY_XMMCMD movdqa, (e0_N + ((%1 + 1) mod 2)), abcd_N + sha1rnds4 abcd, %tok(%strcat(e, %eval(e0_N + (%1 mod 2)))), %1 / 5 - if (k EQ NUM_ROUNDS4 - 1) - nextM = e0_save_N - endif - - MY_sha1nexte (e0_N + ((k + 1) mod 2)), nextM - - if (k GE (4 - pre2)) AND (k LT (NUM_ROUNDS4 - pre2)) - pxor @CatStr(xmm, %(w_regs + ((k + pre2) mod 4))), @CatStr(xmm, %(w_regs + ((k + pre2 - 2) mod 4))) - endif + %assign nextM w_regs + ((%1 + 1) mod 4) - if (k GE (4 - pre1)) AND (k LT (NUM_ROUNDS4 - pre1)) - MY_sha1msg1 (w_regs + ((k + pre1) mod 4)), (w_regs + ((k + pre1 - 3) mod 4)) - endif - - if (k GE (4 - pre2)) AND (k LT (NUM_ROUNDS4 - pre2)) - MY_sha1msg2 (w_regs + ((k + pre2) mod 4)), (w_regs + ((k + pre2 - 1) mod 4)) - endif -endm + %if (%1 == NUM_ROUNDS4 - 1) + %assign nextM e0_save_N + %endif + MY_XMMCMD sha1nexte, (e0_N + ((%1 + 1) mod 2)), nextM -REVERSE_STATE macro - ; abcd ; dcba - ; e0 ; 000e - pshufd abcd, abcd, 01bH ; abcd - pshufd e0, e0, 01bH ; e000 -endm + %if (%1 >= (4 - pre2)) && (%1 < (NUM_ROUNDS4 - pre2)) + MY_XMMCMD pxor, (w_regs + ((%1 + pre2) mod 4)), (w_regs + ((%1 + pre2 - 2) mod 4)) + %endif + %if (%1 >= (4 - pre1)) && (%1 < (NUM_ROUNDS4 - pre1)) + MY_XMMCMD sha1msg1, (w_regs + ((%1 + pre1) mod 4)), (w_regs + ((%1 + pre1 - 3) mod 4)) + %endif + + %if (%1 >= (4 - pre2)) && (%1 < (NUM_ROUNDS4 - pre2)) + MY_XMMCMD sha1msg2, (w_regs + ((%1 + pre2) mod 4)), (w_regs + ((%1 + pre2 - 1) mod 4)) + %endif +%endmacro + + +%macro REVERSE_STATE 0 + ; abcd ; dcba + ; e0 ; 000e + pshufd abcd, abcd, 1BH ; abcd + pshufd e0, e0, 1BH ; e000 +%endmacro MY_PROC Sha1_UpdateBlocks_HW, 3 - MY_PROLOG + MY_PROLOG cmp rNum, 0 - je end_c + je .end_c - movdqu abcd, [rState] ; dcba - movd e0, dword ptr [rState + 16] ; 000e + movdqu abcd, [rState] ; dcba + movd e0, dword [rState + 16] ; 000e REVERSE_STATE - - ifdef x64 + + %if XBITS == 64 LOAD_MASK - endif + %endif - align 16 - nextBlock: + + ALIGN 16 + .nextBlock: movdqa abcd_save, abcd movdqa e0_save, e0 - - ifndef x64 + + %if XBITS == 32 LOAD_MASK - endif - + %endif + LOAD_W 0 LOAD_W 1 LOAD_W 2 LOAD_W 3 - paddd e0, @CatStr(xmm, %(w_regs)) - k = 0 - rept NUM_ROUNDS4 + paddd e0, XMM_REG(w_regs) + + %assign k 0 + %rep NUM_ROUNDS4 RND4 k - k = k + 1 - endm + %assign k k+1 + %endrep paddd abcd, abcd_save - add rData, 64 sub rNum, 1 - jnz nextBlock - + jne .nextBlock + REVERSE_STATE movdqu [rState], abcd - movd dword ptr [rState + 16], e0 - - end_c: -MY_EPILOG + movd dword [rState + 16], e0 + + .end_c: + MY_EPILOG + + + + +[section READONLY] -; _TEXT$SHA1OPT ENDS +ALIGN 16 +.Reverse_Endian_Mask db 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0 -end diff --git a/Asm/x86/Sha256Opt.asm b/Asm/x86/Sha256Opt.asm index bc2f9da24..b7129ef38 100644 --- a/Asm/x86/Sha256Opt.asm +++ b/Asm/x86/Sha256Opt.asm @@ -1,275 +1,233 @@ ; Sha256Opt.asm -- SHA-256 optimized code for SHA-256 x86 hardware instructions ; 2024-06-16 : Igor Pavlov : Public domain -include 7zAsm.asm +%include "7zAsm.inc" MY_ASM_START -; .data -; public K ; we can use external SHA256_K_ARRAY defined in Sha256.c ; but we must guarantee that SHA256_K_ARRAY is aligned for 16-bytes -COMMENT @ -ifdef x64 -K_CONST equ SHA256_K_ARRAY -else -K_CONST equ _SHA256_K_ARRAY -endif -EXTRN K_CONST:xmmword -@ - -CONST SEGMENT READONLY - -align 16 -Reverse_Endian_Mask db 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 - -; COMMENT @ -align 16 -K_CONST \ -DD 0428a2f98H, 071374491H, 0b5c0fbcfH, 0e9b5dba5H -DD 03956c25bH, 059f111f1H, 0923f82a4H, 0ab1c5ed5H -DD 0d807aa98H, 012835b01H, 0243185beH, 0550c7dc3H -DD 072be5d74H, 080deb1feH, 09bdc06a7H, 0c19bf174H -DD 0e49b69c1H, 0efbe4786H, 00fc19dc6H, 0240ca1ccH -DD 02de92c6fH, 04a7484aaH, 05cb0a9dcH, 076f988daH -DD 0983e5152H, 0a831c66dH, 0b00327c8H, 0bf597fc7H -DD 0c6e00bf3H, 0d5a79147H, 006ca6351H, 014292967H -DD 027b70a85H, 02e1b2138H, 04d2c6dfcH, 053380d13H -DD 0650a7354H, 0766a0abbH, 081c2c92eH, 092722c85H -DD 0a2bfe8a1H, 0a81a664bH, 0c24b8b70H, 0c76c51a3H -DD 0d192e819H, 0d6990624H, 0f40e3585H, 0106aa070H -DD 019a4c116H, 01e376c08H, 02748774cH, 034b0bcb5H -DD 0391c0cb3H, 04ed8aa4aH, 05b9cca4fH, 0682e6ff3H -DD 0748f82eeH, 078a5636fH, 084c87814H, 08cc70208H -DD 090befffaH, 0a4506cebH, 0bef9a3f7H, 0c67178f2H -; @ - -CONST ENDS - -; _TEXT$SHA256OPT SEGMENT 'CODE' - -ifndef x64 - .686 - .xmm -endif - -; jwasm-based assemblers for linux and linker from new versions of binutils -; can generate incorrect code for load [ARRAY + offset] instructions. -; 22.00: we load K_CONST offset to (rTable) register to avoid jwasm+binutils problem - rTable equ r0 - ; rTable equ K_CONST - -ifdef x64 - rNum equ REG_ABI_PARAM_2 - if (IS_LINUX eq 0) - LOCAL_SIZE equ (16 * 2) - endif -else - rNum equ r3 - LOCAL_SIZE equ (16 * 1) -endif - -rState equ REG_ABI_PARAM_0 -rData equ REG_ABI_PARAM_1 - - - +; extern SHA256_K_ARRAY +; %define .K_CONST SHA256_K_ARRAY +; jwasm-based assemblers for linux and linker from new versions of binutils +; can generate incorrect code for load [ARRAY + offset] instructions. +; 22.00: we load .K_CONST offset to (rTable) register to avoid jwasm+binutils problem + %define rTable r0 + ; %define rTable .K_CONST -MY_SHA_INSTR macro cmd, a1, a2 - db 0fH, 038H, cmd, (0c0H + a1 * 8 + a2) -endm +%if XBITS == 64 + %define rNum REG_ABI_PARAM_2 + %if ABI == WINDOWS + %define LOCAL_SIZE (16 * 2) + %endif +%else + %define rNum r3 + %define LOCAL_SIZE (16 * 1) +%endif -cmd_sha256rnds2 equ 0cbH -cmd_sha256msg1 equ 0ccH -cmd_sha256msg2 equ 0cdH +%define rState REG_ABI_PARAM_0 +%define rData REG_ABI_PARAM_1 -MY_sha256rnds2 macro a1, a2 - MY_SHA_INSTR cmd_sha256rnds2, a1, a2 -endm -MY_sha256msg1 macro a1, a2 - MY_SHA_INSTR cmd_sha256msg1, a1, a2 -endm -MY_sha256msg2 macro a1, a2 - MY_SHA_INSTR cmd_sha256msg2, a1, a2 -endm -MY_PROLOG macro - ifdef x64 - if (IS_LINUX eq 0) +%macro MY_PROLOG 0 + %if XBITS == 64 + %if ABI == WINDOWS movdqa [r4 + 8], xmm6 movdqa [r4 + 8 + 16], xmm7 sub r4, LOCAL_SIZE + 8 movdqa [r4 ], xmm8 movdqa [r4 + 16], xmm9 - endif - else ; x86 + %endif + %else ; x86 push r3 push r5 mov r5, r4 - NUM_PUSH_REGS equ 2 - PARAM_OFFSET equ (REG_SIZE * (1 + NUM_PUSH_REGS)) - if (IS_CDECL gt 0) + %define NUM_PUSH_REGS 2 + %define PARAM_OFFSET (REG_SIZE * (1 + NUM_PUSH_REGS)) + %if IS_CDECL == 1 mov rState, [r4 + PARAM_OFFSET] mov rData, [r4 + PARAM_OFFSET + REG_SIZE * 1] mov rNum, [r4 + PARAM_OFFSET + REG_SIZE * 2] - else ; fastcall + %else ; fastcall mov rNum, [r4 + PARAM_OFFSET] - endif + %endif and r4, -16 sub r4, LOCAL_SIZE - endif -endm + %endif +%endmacro -MY_EPILOG macro - ifdef x64 - if (IS_LINUX eq 0) +%macro MY_EPILOG 0 + %if XBITS == 64 + %if ABI == WINDOWS movdqa xmm8, [r4] movdqa xmm9, [r4 + 16] add r4, LOCAL_SIZE + 8 movdqa xmm6, [r4 + 8] movdqa xmm7, [r4 + 8 + 16] - endif - else ; x86 + %endif + %else ; x86 mov r4, r5 pop r5 pop r3 - endif + %endif MY_ENDP -endm +%endmacro -msg equ xmm0 -tmp equ xmm0 -state0_N equ 2 -state1_N equ 3 -w_regs equ 4 +%define msg xmm0 +%define tmp xmm0 +%define state0_N 2 +%define state1_N 3 +%define w_regs 4 -state1_save equ xmm1 -state0 equ @CatStr(xmm, %state0_N) -state1 equ @CatStr(xmm, %state1_N) +%define state1_save xmm1 +%define state0 XMM_REG(state0_N) +%define state1 XMM_REG(state1_N) -ifdef x64 - state0_save equ xmm8 - mask2 equ xmm9 -else - state0_save equ [r4] - mask2 equ xmm0 -endif +%if XBITS == 64 + %define state0_save xmm8 + %define mask2 xmm9 +%else + %define state0_save [r4] + %define mask2 xmm0 +%endif -LOAD_MASK macro - movdqa mask2, XMMWORD PTR Reverse_Endian_Mask -endm +%macro LOAD_MASK 0 + movdqa mask2, [.Reverse_Endian_Mask] +%endmacro -LOAD_W macro k:req - movdqu @CatStr(xmm, %(w_regs + k)), [rData + (16 * (k))] - pshufb @CatStr(xmm, %(w_regs + k)), mask2 -endm +%macro LOAD_W 1 + movdqu XMM_REG(w_regs + %1), [rData + 16 * %1] + pshufb XMM_REG(w_regs + %1), mask2 +%endmacro ; pre1 <= 4 && pre2 >= 1 && pre1 > pre2 && (pre1 - pre2) <= 1 -pre1 equ 3 -pre2 equ 2 - +%define pre1 3 +%define pre2 2 -RND4 macro k - movdqa msg, xmmword ptr [rTable + (k) * 16] - paddd msg, @CatStr(xmm, %(w_regs + ((k + 0) mod 4))) - MY_sha256rnds2 state0_N, state1_N +%macro RND4 1 + movdqa msg, [rTable + (%1) * 16] + paddd msg, XMM_REG(w_regs + ((%1 + 0) mod 4)) + sha256rnds2 state0, state1 pshufd msg, msg, 0eH - if (k GE (4 - pre1)) AND (k LT (16 - pre1)) + %if (%1 >= (4 - pre1)) && (%1 < (16 - pre1)) ; w4[0] = msg1(w4[-4], w4[-3]) - MY_sha256msg1 (w_regs + ((k + pre1) mod 4)), (w_regs + ((k + pre1 - 3) mod 4)) - endif - - MY_sha256rnds2 state1_N, state0_N + sha256msg1 XMM_REG(w_regs + ((%1 + pre1) mod 4)), XMM_REG(w_regs + ((%1 + pre1 - 3) mod 4)) + %endif + + sha256rnds2 state1, state0 - if (k GE (4 - pre2)) AND (k LT (16 - pre2)) - movdqa tmp, @CatStr(xmm, %(w_regs + ((k + pre2 - 1) mod 4))) - palignr tmp, @CatStr(xmm, %(w_regs + ((k + pre2 - 2) mod 4))), 4 - paddd @CatStr(xmm, %(w_regs + ((k + pre2) mod 4))), tmp + %if (%1 >= (4 - pre2)) && (%1 < (16 - pre2)) + movdqa tmp, XMM_REG(w_regs + ((%1 + pre2 - 1) mod 4)) + palignr tmp, XMM_REG(w_regs + ((%1 + pre2 - 2) mod 4)), 4 + paddd XMM_REG(w_regs + ((%1 + pre2) mod 4)), tmp ; w4[0] = msg2(w4[0], w4[-1]) - MY_sha256msg2 %(w_regs + ((k + pre2) mod 4)), %(w_regs + ((k + pre2 - 1) mod 4)) - endif -endm + sha256msg2 XMM_REG(w_regs + ((%1 + pre2) mod 4)), XMM_REG(w_regs + ((%1 + pre2 - 1) mod 4)) + %endif +%endmacro -REVERSE_STATE macro +%macro REVERSE_STATE 0 ; state0 ; dcba ; state1 ; hgfe - pshufd tmp, state0, 01bH ; abcd - pshufd state0, state1, 01bH ; efgh + pshufd tmp, state0, 1BH ; abcd + pshufd state0, state1, 1BH ; efgh movdqa state1, state0 ; efgh punpcklqdq state0, tmp ; cdgh punpckhqdq state1, tmp ; abef -endm +%endmacro MY_PROC Sha256_UpdateBlocks_HW, 3 - MY_PROLOG + MY_PROLOG - lea rTable, [K_CONST] + lea rTable, [.K_CONST] cmp rNum, 0 - je end_c + je .end_c movdqu state0, [rState] ; dcba movdqu state1, [rState + 16] ; hgfe REVERSE_STATE - ifdef x64 + %if XBITS == 64 LOAD_MASK - endif + %endif align 16 - nextBlock: + .nextBlock: movdqa state0_save, state0 movdqa state1_save, state1 - - ifndef x64 + + %if XBITS == 32 LOAD_MASK - endif - + %endif + LOAD_W 0 LOAD_W 1 LOAD_W 2 LOAD_W 3 - - k = 0 - rept 16 + %assign k 0 + %rep 16 RND4 k - k = k + 1 - endm + %assign k k+1 + %endrep paddd state0, state0_save paddd state1, state1_save add rData, 64 sub rNum, 1 - jnz nextBlock - + jnz .nextBlock + REVERSE_STATE movdqu [rState], state0 movdqu [rState + 16], state1 - - end_c: -MY_EPILOG -; _TEXT$SHA256OPT ENDS + .end_c: + MY_EPILOG + + + + +[section READONLY] + +ALIGN 16 +.Reverse_Endian_Mask db 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 + +ALIGN 16 +.K_CONST: + DD 0428a2f98H, 071374491H, 0b5c0fbcfH, 0e9b5dba5H + DD 03956c25bH, 059f111f1H, 0923f82a4H, 0ab1c5ed5H + DD 0d807aa98H, 012835b01H, 0243185beH, 0550c7dc3H + DD 072be5d74H, 080deb1feH, 09bdc06a7H, 0c19bf174H + DD 0e49b69c1H, 0efbe4786H, 00fc19dc6H, 0240ca1ccH + DD 02de92c6fH, 04a7484aaH, 05cb0a9dcH, 076f988daH + DD 0983e5152H, 0a831c66dH, 0b00327c8H, 0bf597fc7H + DD 0c6e00bf3H, 0d5a79147H, 006ca6351H, 014292967H + DD 027b70a85H, 02e1b2138H, 04d2c6dfcH, 053380d13H + DD 0650a7354H, 0766a0abbH, 081c2c92eH, 092722c85H + DD 0a2bfe8a1H, 0a81a664bH, 0c24b8b70H, 0c76c51a3H + DD 0d192e819H, 0d6990624H, 0f40e3585H, 0106aa070H + DD 019a4c116H, 01e376c08H, 02748774cH, 034b0bcb5H + DD 0391c0cb3H, 04ed8aa4aH, 05b9cca4fH, 0682e6ff3H + DD 0748f82eeH, 078a5636fH, 084c87814H, 08cc70208H + DD 090befffaH, 0a4506cebH, 0bef9a3f7H, 0c67178f2H -end diff --git a/Asm/x86/Sort.asm b/Asm/x86/Sort.asm index 517c6152c..834b9e8e5 100644 --- a/Asm/x86/Sort.asm +++ b/Asm/x86/Sort.asm @@ -1,256 +1,269 @@ ; SortTest.asm -- ASM version of HeapSort() function ; Igor Pavlov : Public domain -include ../../../../Asm/x86/7zAsm.asm +%include "7zAsm.inc" MY_ASM_START -ifndef Z7_SORT_ASM_USE_SEGMENT -if (IS_LINUX gt 0) - ; Z7_SORT_ASM_USE_SEGMENT equ 1 -else - ; Z7_SORT_ASM_USE_SEGMENT equ 1 -endif -endif - -ifdef Z7_SORT_ASM_USE_SEGMENT -_TEXT$Z7_SORT SEGMENT ALIGN(64) 'CODE' -MY_ALIGN macro num:req - align num -endm -else -MY_ALIGN macro num:req +%ifndef Z7_SORT_ASM_USE_SEGMENT +; %define Z7_SORT_ASM_USE_SEGMENT 1 +%endif + +%ifdef Z7_SORT_ASM_USE_SEGMENT +%macro MY_ALIGN 1 + ALIGN %1 +%endmacro +%else +%macro MY_ALIGN 1 ; We expect that ".text" is aligned for 16-bytes. ; So we don't need large alignment inside our function. - align 16 -endm -endif + ALIGN 16 +%endmacro +%endif -MY_ALIGN_16 macro +%macro MY_ALIGN_16 0 MY_ALIGN 16 -endm +%endmacro -MY_ALIGN_32 macro +%macro MY_ALIGN_32 0 MY_ALIGN 32 -endm +%endmacro -MY_ALIGN_64 macro +%macro MY_ALIGN_64 0 MY_ALIGN 64 -endm +%endmacro + + +%define STEP_NONE_N 0 +%define STEP_1_N 1 +%define STEP_2_N 2 +%define STEP_BRANCH_N 3 + -ifdef x64 +%if XBITS == 64 -NUM_PREFETCH_LEVELS equ 3 ; to prefetch 1x 64-bytes line (is good for most cases) -; NUM_PREFETCH_LEVELS equ 4 ; to prefetch 2x 64-bytes lines (better for big arrays) +%define NUM_PREFETCH_LEVELS 3 ; to prefetch 1x 64-bytes line (is good for most cases) +;%define NUM_PREFETCH_LEVELS 4 ; to prefetch 2x 64-bytes lines (better for big arrays) -acc equ x0 -k equ r0 -k_x equ x0 +%define acc x0 +%define k r0 +%define k_x acc -p equ r1 +%define p r1 -s equ r2 -s_x equ x2 +%define s r2 +%define s_x x2 -a0 equ x3 -t0 equ a0 +%define a0 x3 +%define t0 a0 -a3 equ x5 -qq equ a3 +%define a3 x5 +%define qq a3 -a1 equ x6 -t1 equ a1 -t1_r equ r6 +%define a1 x6 +%define t1 a1 +%define t1_r r6 -a2 equ x7 -t2 equ a2 +%define a2 x7 +%define t2 a2 -i equ r8 -e0 equ x8 +%define i r8 +%define e0 x8 -e1 equ x9 +%define e1 x9 -num_last equ r10 -num_last_x equ x10 +%define num_last r10 +%define num_last_x x10 -next4_lim equ r11 -pref_lim equ r12 +%define next4_lim r11 +%define pref_lim r12 -SORT_2_WITH_TEMP_REG macro b0, b1, temp_reg - mov temp_reg, b0 - cmp b0, b1 - cmovae b0, b1 ; min - cmovae b1, temp_reg ; max -endm +%macro SORT_2_WITH_TEMP_REG 3 ; b0, b1, temp_reg + mov %3, %1 + cmp %1, %2 + cmovae %1, %2 ; min + cmovae %2, %3 ; max +%endmacro -SORT macro b0, b1 - SORT_2_WITH_TEMP_REG b0, b1, acc -endm +%macro SORT 2 ; b0, b1 + SORT_2_WITH_TEMP_REG %1, %2, acc +%endmacro -LOAD macro dest:req, index:req - mov dest, [p + 4 * index] -endm +%macro LOAD 2 ; dest, index + mov %1, [p + 4 * %2] +%endmacro -STORE macro reg:req, index:req - mov [p + 4 * index], reg -endm +%macro STORE 2 ; reg, index + mov [p + 4 * %2], %1 +%endmacro -if (NUM_PREFETCH_LEVELS gt 3) - num_prefetches equ (1 SHL (NUM_PREFETCH_LEVELS - 3)) -else - num_prefetches equ 1 -endif +%if NUM_PREFETCH_LEVELS > 3 + %define num_prefetches (1 << (NUM_PREFETCH_LEVELS - 3)) +%else + %define num_prefetches 1 +%endif -PREFETCH_OP macro offs - cur_offset = 7 * 4 ; it's average offset in 64-bytes cache line. - ; cur_offset = 0 ; we can use zero offset, if we are sure that array is aligned for 64-bytes. - rept num_prefetches - if 1 - prefetcht0 byte ptr [p + offs + cur_offset] - else - mov pref_x, dword ptr [p + offs + cur_offset] - endif - cur_offset = cur_offset + 64 - endm -endm +%macro PREFETCH_OP 1 ; offs + %assign cur_offset 7 * 4 ; it's average offset in 64-bytes cache line. +; %assign cur_offset 0 ; we can use zero offset, if we are sure that array is aligned for 64-bytes. + %rep num_prefetches + %if 1 + prefetcht0 byte [p + %1 + cur_offset] + %else + mov pref_x, dword [p + %1 + cur_offset] + %endif + %assign cur_offset cur_offset+64 + %endrep +%endmacro -PREFETCH_MY macro -if 1 - if 1 +%macro PREFETCH_MY 0 + %if 1 + %if 1 shl k, NUM_PREFETCH_LEVELS + 3 - else + %else ; we delay prefetch instruction to improve main loads shl k, NUM_PREFETCH_LEVELS shl k, 3 ; shl k, 0 - endif + %endif PREFETCH_OP k -elseif 1 + %elif 1 shl k, 3 - PREFETCH_OP k * (1 SHL NUM_PREFETCH_LEVELS) ; change it -endif -endm + PREFETCH_OP k * (1 << NUM_PREFETCH_LEVELS) ; change it + %endif +%endmacro -STEP_1 macro exit_label, prefetch_macro -use_cmov_1 equ 1 ; set 1 for cmov, but it's slower in some cases - ; set 0 for LOAD after adc s, 0 +%macro STEP_1 2 ; exit_label, use_prefetch + %define use_cmov_1 1 ; set 1 for cmov, but it's slower in some cases + ; set 0 for LOAD after adc s, 0 cmp t0, t1 - if use_cmov_1 + %if use_cmov_1 > 0 cmovb t0, t1 ; STORE t0, k - endif + %endif adc s, 0 - if use_cmov_1 eq 0 + %if use_cmov_1 == 0 LOAD t0, s - endif + %endif cmp qq, t0 - jae exit_label - if 1 ; use_cmov_1 eq 0 + jae %1 + %if 1 ; use_cmov_1 == 0 STORE t0, k - endif - prefetch_macro + %endif + %if %2 > 0 + PREFETCH_MY + %endif mov t0, [p + s * 8] mov t1, [p + s * 8 + 4] mov k, s add s, s ; slower for some cpus - ; lea s, dword ptr [s + s] ; slower for some cpus + ; lea s, dword [s + s] ; slower for some cpus ; shl s, 1 ; faster for some cpus - ; lea s, dword ptr [s * 2] ; faster for some cpus - rept 0 ; 1000 for debug : 0 for normal + ; lea s, dword [s * 2] ; faster for some cpus + %rep 0 ; 1000 for debug : 0 for normal ; number of calls in generate_stage : ~0.6 of number of items shl k, 0 - endm -endm + %endrep +%endmacro -STEP_2 macro exit_label, prefetch_macro -use_cmov_2 equ 0 ; set 1 for cmov, but it's slower in some cases - ; set 0 for LOAD after adc s, 0 +%macro STEP_2 2 ; exit_label, use_prefetch + %define use_cmov_2 0 ; set 1 for cmov, but it's slower in some cases + ; set 0 for LOAD after adc s, 0 cmp t0, t1 - if use_cmov_2 + %if use_cmov_2 == 1 mov t2, t0 cmovb t2, t1 ; STORE t2, k - endif + %endif mov t0, [p + s * 8] mov t1, [p + s * 8 + 4] cmovb t0, [p + s * 8 + 8] cmovb t1, [p + s * 8 + 12] adc s, 0 - if use_cmov_2 eq 0 + %if use_cmov_2 == 0 LOAD t2, s - endif + %endif cmp qq, t2 - jae exit_label - if 1 ; use_cmov_2 eq 0 + jae %1 + %if 1 ; use_cmov_2 == 0 STORE t2, k - endif - prefetch_macro + %endif + %if %2 > 0 + PREFETCH_MY + %endif mov k, s ; add s, s ; lea s, [s + s] shl s, 1 ; lea s, [s * 2] -endm +%endmacro -MOVE_SMALLEST_UP macro STEP, use_prefetch, num_unrolls - LOCAL exit_1, exit_2, leaves, opt_loop, last_nodes +%macro STEP_X 3 ; STEP_N, exit_label, use_prefetch + %if %1 == STEP_1_N + STEP_1 %2, %3 + %elif %1 == STEP_2_N + STEP_2 %2, %3 + %endif +%endmacro + +%macro MOVE_SMALLEST_UP 3 ; STEP_N, use_prefetch, num_unrolls ; s == k * 2 ; t0 == (p)[s] ; t1 == (p)[s + 1] cmp k, next4_lim - jae leaves + jae %%leaves - rept num_unrolls - STEP exit_2 + %rep %3 + STEP_X %1, %%exit_2, 0 cmp k, next4_lim - jae leaves - endm - - if use_prefetch - prefetch_macro equ PREFETCH_MY - pref_lim_2 equ pref_lim - ; lea pref_lim, dword ptr [num_last + 1] + jae %%leaves + %endrep + + %if %2 > 0 + %define pref_lim_2 pref_lim + ; lea pref_lim, dword [num_last + 1] ; shr pref_lim, NUM_PREFETCH_LEVELS + 1 cmp k, pref_lim_2 - jae last_nodes - else - prefetch_macro equ - pref_lim_2 equ next4_lim - endif - + jae %%last_nodes + %else + %define pref_lim_2 next4_lim + %endif + MY_ALIGN_16 -opt_loop: - STEP exit_2, prefetch_macro +%%opt_loop: + STEP_X %1, %%exit_2, %2 cmp k, pref_lim_2 - jb opt_loop + jb %%opt_loop -last_nodes: +%%last_nodes: ; k >= pref_lim_2 ; 2 cases are possible: ; case-1: num_after_prefetch_levels == 0 && next4_lim = pref_lim_2 ; case-2: num_after_prefetch_levels == NUM_PREFETCH_LEVELS - 1 && ; next4_lim = pref_lim_2 / (NUM_PREFETCH_LEVELS - 1) - if use_prefetch - yyy = NUM_PREFETCH_LEVELS - 1 - while yyy - yyy = yyy - 1 - STEP exit_2 - if yyy + %if %2 > 0 + %assign yyy NUM_PREFETCH_LEVELS - 1 + %rep NUM_PREFETCH_LEVELS + %assign yyy yyy-1 + STEP_X %1, %%exit_2, 0 + %if yyy != 0 cmp k, next4_lim - jae leaves - endif - endm - endif - -leaves: + jae %%leaves + %else + %exitrep + %endif + %endrep + %endif + +%%leaves: ; k >= next4_lim == (num_last + 1) / 4 must be provided by previous code. ; we have 2 nodes in (s) level : always ; we can have some nodes in (s * 2) level : low probability case @@ -271,14 +284,14 @@ leaves: ; s * 2 < num_last : (s) node has 2 leaf childs. We provide (s * 4 > num_last) ; we check for (s * 2 > num_last) before "cmp qq, t0" check, because ; we will replace conditional jump with cmov instruction later. - lea t1_r, dword ptr [s + s] + lea t1_r, dword [s + s] cmp t1_r, num_last - ja exit_1 ; if (s * 2 > num_last), we have no childs : it's high probability branch + ja %%exit_1 ; if (s * 2 > num_last), we have no childs : it's high probability branch ; it's low probability branch ; s * 2 <= num_last cmp qq, t0 - jae exit_2 + jae %%exit_2 ; qq < t0, so we go to next level ; we check 1 or 2 childs in next level @@ -295,72 +308,72 @@ leaves: adc s, 0 @@: STORE t0, k -exit_1: +%%exit_1: ; t0 == (p)[s], s / 2 == k : (s) is index of max item from (p)[k * 2], (p)[k * 2 + 1] cmp qq, t0 cmovb k, s -exit_2: +%%exit_2: STORE qq, k -endm +%endmacro -ifdef Z7_SORT_ASM_USE_SEGMENT -; MY_ALIGN_64 -else +%ifdef Z7_SORT_ASM_USE_SEGMENT + ; MY_ALIGN_64 + [section .text$Z7_SORT align=64 exec] +%else MY_ALIGN_16 -endif +%endif MY_PROC HeapSort, 2 - -if (IS_LINUX gt 0) + %if ABI == LINUX mov p, REG_ABI_PARAM_0 ; r1 <- r7 : linux -endif + %endif mov num_last, REG_ABI_PARAM_1 ; r10 <- r6 : linux ; r10 <- r2 : win64 cmp num_last, 2 - jb end_1 - + jb .end_1 + ; MY_PUSH_PRESERVED_ABI_REGS MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 push r12 - + cmp num_last, 4 - ja sort_5 - + ja .sort_5 + LOAD a0, 0 LOAD a1, 1 SORT a0, a1 cmp num_last, 3 - jb end_2 - + jb .end_2 + LOAD a2, 2 - je sort_3 - + je .sort_3 + LOAD a3, 3 SORT a2, a3 SORT a1, a3 STORE a3, 3 -sort_3: +.sort_3: SORT a0, a2 SORT a1, a2 STORE a2, 2 - jmp end_2 - -sort_5: + jmp .end_2 + +.sort_5: ; (num_last > 4) is required here - ; if (num_last >= 6) : we will use optimized loop for leaf nodes loop_down_1 + ; if (num_last >= 6) : we will use optimized loop for leaf nodes .loop_down_1 mov next4_lim, num_last shr next4_lim, 2 - + dec num_last mov k, num_last shr k, 1 mov i, num_last shr i, 2 test num_last, 1 - jnz size_even + jnz .size_even ; ODD number of items. So we compare parent with single child LOAD t1, num_last @@ -370,23 +383,23 @@ sort_5: STORE t0, k dec k -size_even: +.size_even: cmp k, i - jbe loop_down ; jump for num_last == 4 case + jbe .loop_down ; jump for num_last == 4 case -if 0 ; 1 for debug +%if 0 ; 1 for debug mov r15, k mov r14d, 1 ; 100 -loop_benchmark: -endif +.loop_benchmark: +%endif ; optimized loop for leaf nodes: mov t0, [p + k * 8] mov t1, [p + k * 8 + 4] MY_ALIGN_16 -loop_down_1: +.loop_down_1: ; we compare parent with max of childs: - ; lea s, dword ptr [2 * k] + ; lea s, dword [2 * k] mov s, k cmp t0, t1 cmovb t0, t1 @@ -401,61 +414,62 @@ loop_down_1: mov t1, [p + k * 8 + 4] STORE t2, s cmp k, i - jne loop_down_1 + jne .loop_down_1 -if 0 ; 1 for debug +%if 0 ; 1 for debug mov k, r15 dec r14d - jnz loop_benchmark - ; jmp end_debug -endif - + jnz .loop_benchmark + ; jmp .end_debug +%endif + MY_ALIGN_16 -loop_down: +.loop_down: mov t0, [p + i * 8] mov t1, [p + i * 8 + 4] LOAD qq, i mov k, i - lea s, dword ptr [i + i] - ; jmp end_debug - DOWN_use_prefetch equ 0 - DOWN_num_unrolls equ 0 - MOVE_SMALLEST_UP STEP_1, DOWN_use_prefetch, DOWN_num_unrolls + lea s, dword [i + i] + ; jmp .end_debug + + %assign DOWN_use_prefetch 0 + %assign DOWN_num_unrolls 0 + MOVE_SMALLEST_UP STEP_1_N, DOWN_use_prefetch, DOWN_num_unrolls sub i, 1 - jnb loop_down + jnb .loop_down - ; jmp end_debug + ; jmp .end_debug LOAD e0, 0 LOAD e1, 1 - LEVEL_3_LIMIT equ 8 ; 8 is default, but 7 also can work + %define LEVEL_3_LIMIT 8 ; 8 is default, but 7 also can work cmp num_last, LEVEL_3_LIMIT + 1 - jb main_loop_sort_5 + jb .main_loop_sort_5 MY_ALIGN_16 -main_loop_sort: +.main_loop_sort: ; num_last > LEVEL_3_LIMIT ; p[size--] = p[0]; LOAD qq, num_last STORE e0, num_last mov e0, e1 - + mov next4_lim, num_last shr next4_lim, 2 mov pref_lim, num_last shr pref_lim, NUM_PREFETCH_LEVELS + 1 - + dec num_last -if 0 ; 1 for debug +%if 0 ; 1 for debug ; that optional optimization can improve the performance, if there are identical items in array ; 3 times improvement : if all items in array are identical ; 20% improvement : if items are different for 1 bit only ; 1-10% improvement : if items are different for (2+) bits ; no gain : if items are different cmp qq, e1 - jae next_iter_main -endif + jae .next_iter_main +%endif LOAD e1, 2 LOAD t0, 3 mov k_x, 2 @@ -468,24 +482,24 @@ endif adc k_x, 0 ; (qq <= e1), because the tree is correctly sorted ; also here we could check (qq >= e1) or (qq == e1) for faster exit - lea s, dword ptr [k + k] - MAIN_use_prefetch equ 1 - MAIN_num_unrolls equ 0 - MOVE_SMALLEST_UP STEP_2, MAIN_use_prefetch, MAIN_num_unrolls + lea s, dword [k + k] + %assign MAIN_use_prefetch 1 + %assign MAIN_num_unrolls 0 + MOVE_SMALLEST_UP STEP_2_N, MAIN_use_prefetch, MAIN_num_unrolls -next_iter_main: +.next_iter_main: cmp num_last, LEVEL_3_LIMIT - jne main_loop_sort + jne .main_loop_sort ; num_last == LEVEL_3_LIMIT -main_loop_sort_5: +.main_loop_sort_5: ; 4 <= num_last <= LEVEL_3_LIMIT ; p[size--] = p[0]; LOAD qq, num_last STORE e0, num_last mov e0, e1 dec num_last_x - + LOAD e1, 2 LOAD t0, 3 mov k_x, 2 @@ -493,26 +507,26 @@ main_loop_sort_5: cmovb e1, t0 adc k_x, 0 - lea s_x, dword ptr [k * 2] + lea s_x, dword [k * 2] cmp s_x, num_last_x - ja exit_2 + ja .exit_2 mov t0, [p + k * 8] - je exit_1 + je .exit_1 ; s < num_last mov t1, [p + k * 8 + 4] cmp t0, t1 cmovb t0, t1 adc s_x, 0 -exit_1: +.exit_1: STORE t0, k cmp qq, t0 cmovb k_x, s_x -exit_2: +.exit_2: STORE qq, k cmp num_last_x, 3 - jne main_loop_sort_5 + jne .main_loop_sort_5 ; num_last == 3 (real_size == 4) LOAD a0, 2 @@ -520,88 +534,84 @@ exit_2: STORE e1, 2 STORE e0, 3 SORT a0, a1 -end_2: +.end_2: STORE a0, 0 STORE a1, 1 -; end_debug: +; .end_debug: ; MY_POP_PRESERVED_ABI_REGS pop r12 MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 -end_1: -MY_ENDP +.end_1: + MY_ENDP -else +%else ; ------------ x86 32-bit ------------ -ifdef x64 -IS_CDECL = 0 -endif +%define acc x0 +%define k r0 +%define k_x acc -acc equ x0 -k equ r0 -k_x equ acc +%define p r1 -p equ r1 +%define num_last r2 +%define num_last_x x2 -num_last equ r2 -num_last_x equ x2 +%define a0 x3 +%define t0 a0 -a0 equ x3 -t0 equ a0 +%define a3 x5 +%define i r5 +%define e0 a3 -a3 equ x5 -i equ r5 -e0 equ a3 +%define a1 x6 +%define qq a1 -a1 equ x6 -qq equ a1 - -a2 equ x7 -s equ r7 -s_x equ a2 +%define a2 x7 +%define s r7 +%define s_x a2 -SORT macro b0, b1 - cmp b1, b0 +%macro SORT 2 ; b0, b1 + cmp %2, %1 jae @F - if 1 - xchg b0, b1 - else - mov acc, b0 - mov b0, b1 ; min - mov b1, acc ; max - endif + %if 1 + xchg %1, %2 + %else + mov acc, %1 + mov %1, %2 ; min + mov %2, acc ; max + %endif @@: -endm +%endmacro -LOAD macro dest:req, index:req - mov dest, [p + 4 * index] -endm +%macro LOAD 2 ; dest, index + mov %1, [p + 4 * %2] +%endmacro -STORE macro reg:req, index:req - mov [p + 4 * index], reg -endm +%macro STORE 2 ; reg, index + mov [p + 4 * %2], %1 +%endmacro -STEP_1 macro exit_label +%macro STEP_1 1 ; exit_label mov t0, [p + k * 8] cmp t0, [p + k * 8 + 4] adc s, 0 LOAD t0, s STORE t0, k ; we lookahed stooring for most expected branch cmp qq, t0 - jae exit_label + jae %1 ; STORE t0, k ; use if mov k, s add s, s - ; lea s, dword ptr [s + s] + ; lea s, dword [s + s] ; shl s, 1 - ; lea s, dword ptr [s * 2] -endm + ; lea s, dword [s * 2] +%endmacro -STEP_BRANCH macro exit_label +%macro STEP_BRANCH 1 ; exit_label mov t0, [p + k * 8] cmp t0, [p + k * 8 + 4] jae @F @@ -609,112 +619,119 @@ STEP_BRANCH macro exit_label mov t0, [p + k * 8 + 4] @@: cmp qq, t0 - jae exit_label + jae %1 STORE t0, k mov k, s add s, s -endm +%endmacro + +%macro STEP_X 2 ; STEP_N, label + %if %1 == STEP_1_N + STEP_1 %2 + %elif %1 == STEP_BRANCH_N + STEP_BRANCH %2 + %endif +%endmacro -MOVE_SMALLEST_UP macro STEP, num_unrolls, exit_2 - LOCAL leaves, opt_loop, single +%macro MOVE_SMALLEST_UP 3 ; STEP_N, num_unrolls, exit_2 ; s == k * 2 - rept num_unrolls + %rep %2 cmp s, num_last - jae leaves - STEP_1 exit_2 - endm + jae %%leaves + STEP_1 %3 + %endrep cmp s, num_last - jb opt_loop + jb %%opt_loop -leaves: +%%leaves: ; (s >= num_last) - jne exit_2 -single: + jne %3 +%%single: ; (s == num_last) mov t0, [p + k * 8] cmp qq, t0 - jae exit_2 + jae %3 STORE t0, k mov k, s - jmp exit_2 - + jmp %3 + MY_ALIGN_16 -opt_loop: - STEP exit_2 +%%opt_loop: + STEP_X %1, %3 cmp s, num_last - jb opt_loop - je single -exit_2: + jb %%opt_loop + je %%single +%3: STORE qq, k -endm +%endmacro -ifdef Z7_SORT_ASM_USE_SEGMENT +%ifdef Z7_SORT_ASM_USE_SEGMENT ; MY_ALIGN_64 -else +%else MY_ALIGN_16 -endif +%endif MY_PROC HeapSort, 2 - ifdef x64 - if (IS_LINUX gt 0) + %if XBITS == 64 + %if ABI == LINUX mov num_last, REG_ABI_PARAM_1 ; r2 <- r6 : linux mov p, REG_ABI_PARAM_0 ; r1 <- r7 : linux - endif - elseif (IS_CDECL gt 0) + %endif + %elif IS_CDECL > 0 mov num_last, [r4 + REG_SIZE * 2] mov p, [r4 + REG_SIZE * 1] - endif + %endif cmp num_last, 2 - jb end_1 + jb .end_1 MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 - + cmp num_last, 4 - ja sort_5 - + ja .sort_5 + LOAD a0, 0 LOAD a1, 1 SORT a0, a1 cmp num_last, 3 - jb end_2 - + jb .end_2 + LOAD a2, 2 - je sort_3 - + je .sort_3 + LOAD a3, 3 SORT a2, a3 SORT a1, a3 STORE a3, 3 -sort_3: +.sort_3: SORT a0, a2 SORT a1, a2 STORE a2, 2 - jmp end_2 - -sort_5: + jmp .end_2 + +.sort_5: ; num_last > 4 - lea i, dword ptr [num_last - 2] + lea i, dword [num_last - 2] dec num_last test i, 1 - jz loop_down + jz .loop_down ; single child mov t0, [p + num_last * 4] mov qq, [p + num_last * 2] dec i cmp qq, t0 - jae loop_down + jae .loop_down mov [p + num_last * 2], t0 mov [p + num_last * 4], qq - + MY_ALIGN_16 -loop_down: +.loop_down: mov t0, [p + i * 4] cmp t0, [p + i * 4 + 4] mov k, i @@ -722,21 +739,21 @@ loop_down: adc k, 0 LOAD t0, k cmp qq, t0 - jae down_next + jae .down_next mov [p + i * 2], t0 - lea s, dword ptr [k + k] - - DOWN_num_unrolls equ 0 - MOVE_SMALLEST_UP STEP_1, DOWN_num_unrolls, down_exit_label -down_next: + lea s, dword [k + k] + + %assign DOWN_num_unrolls 0 + MOVE_SMALLEST_UP STEP_1_N, DOWN_num_unrolls, .down_exit_label +.down_next: sub i, 2 - jnb loop_down - ; jmp end_debug + jnb .loop_down + ; jmp .end_debug LOAD e0, 0 MY_ALIGN_16 -main_loop_sort: +.main_loop_sort: ; num_last > 3 mov t0, [p + 2 * 4] cmp t0, [p + 3 * 4] @@ -749,25 +766,25 @@ main_loop_sort: LOAD t0, s dec num_last cmp qq, t0 - jae main_exit_label + jae .main_exit_label STORE t0, 1 mov k, s add s, s - if 1 + %if 1 ; for branch data prefetch mode : ; it's faster for large arrays : larger than (1 << 13) items. - MAIN_num_unrolls equ 10 - STEP_LOOP equ STEP_BRANCH - else - MAIN_num_unrolls equ 0 - STEP_LOOP equ STEP_1 - endif - - MOVE_SMALLEST_UP STEP_LOOP, MAIN_num_unrolls, main_exit_label - - ; jmp end_debug + %assign MAIN_num_unrolls 10 + %assign STEP_LOOP_N STEP_BRANCH_N + %else + %assign MAIN_num_unrolls 0 + %assign STEP_LOOP_N STEP_1_N + %endif + + MOVE_SMALLEST_UP STEP_LOOP_N, MAIN_num_unrolls, .main_exit_label + + ; jmp .end_debug cmp num_last, 3 - jne main_loop_sort + jne .main_loop_sort ; num_last == 3 (real_size == 4) LOAD a0, 2 @@ -776,21 +793,18 @@ main_loop_sort: STORE e0, 3 ; e0 is alias for a3 STORE a2, 2 SORT a0, a1 -end_2: +.end_2: STORE a0, 0 STORE a1, 1 -; end_debug: +; .end_debug: MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 -end_1: -MY_ENDP +.end_1: + MY_ENDP -endif +%endif -ifdef Z7_SORT_ASM_USE_SEGMENT -_TEXT$Z7_SORT ENDS -endif -if 0 +%if 0 LEA_IS_D8 (R64) [R2 * 4 + 16] Lat : TP 2 : 1 : adl-e @@ -855,6 +869,5 @@ PREFETCHNTA : fetch data into non-temporal cache close to the processor, minimiz L3 with fast replacement: Xeon Processors based on Nehalem, Westmere, Sandy Bridge, ... PREFETCHT0 : fetch data into all cache levels. PREFETCHT1 : fetch data into L2 and L3 -endif +%endif -end diff --git a/Asm/x86/XzCrc64Opt.asm b/Asm/x86/XzCrc64Opt.asm index 283424c62..a1095a4c8 100644 --- a/Asm/x86/XzCrc64Opt.asm +++ b/Asm/x86/XzCrc64Opt.asm @@ -1,103 +1,106 @@ ; XzCrc64Opt.asm -- CRC64 calculation : optimized version ; 2023-12-08 : Igor Pavlov : Public domain -include 7zAsm.asm +%include "7zAsm.inc" MY_ASM_START -NUM_WORDS equ 3 +%define NUM_WORDS 3 -if (NUM_WORDS lt 1) or (NUM_WORDS gt 64) -.err -endif +%if (NUM_WORDS < 1) || (NUM_WORDS > 64) + %fatal +%endif -NUM_SKIP_BYTES equ ((NUM_WORDS - 2) * 4) +%define NUM_SKIP_BYTES ((NUM_WORDS - 2) * 4) -MOVZXLO macro dest:req, src:req - movzx dest, @CatStr(src, _L) -endm +; ALIGN_MASK is 3 or 7 bytes alignment: +%define ALIGN_MASK (7 - (NUM_WORDS & 1) * 4) -MOVZXHI macro dest:req, src:req - movzx dest, @CatStr(src, _H) -endm +%macro CRC_1 5 ; op, dest, src, t, word_index + %assign n %find(%3, x0,x1,x2,x3,x4,x5,x6,x7) + %if n == 0 + %fatal + %else + ; x ==> r + %1 %2, [rT + %tok(%strcat('r', %eval(n-1))) * 8 + 0800h * (%4) + (%5) * 4] + %endif +%endmacro -ifdef x64 -rD equ r11 -rN equ r10 -rT equ r9 - -CRC_OP macro op:req, dest:req, src:req, t:req - op dest, QWORD PTR [rT + @CatStr(src, _R) * 8 + 0800h * (t)] -endm - -CRC_XOR macro dest:req, src:req, t:req - CRC_OP xor, dest, src, t -endm - -CRC_MOV macro dest:req, src:req, t:req - CRC_OP mov, dest, src, t -endm - -CRC1b macro - movzx x6, BYTE PTR [rD] +%if XBITS == 64 + +%define rD r11 +%define rN r10 +%define rT r9 + + +%macro CRC_OP 4 + CRC_1 %1, %2, %3, %4, 0 +%endmacro + +%macro CRC_XOR 3 ; dest, src, t + CRC_OP xor, %1, %2, %3 +%endmacro + +%macro CRC_MOV 3 ; dest, src, t + CRC_OP mov, %1, %2, %3 +%endmacro + +%macro CRC1b 0 + movzx x6, byte [rD] inc rD MOVZXLO x3, x0 xor x6, x3 shr r0, 8 CRC_XOR r0, x6, 0 dec rN -endm - +%endmacro -; ALIGN_MASK is 3 or 7 bytes alignment: -ALIGN_MASK equ (7 - (NUM_WORDS and 1) * 4) -if NUM_WORDS eq 1 +%if NUM_WORDS == 1 -src_rN_offset equ 4 +%define src_rN_offset 4 ; + 4 for prefetching next 4-bytes after current iteration -NUM_BYTES_LIMIT equ (NUM_WORDS * 4 + 4) -SRCDAT4 equ DWORD PTR [rN + rD * 1] +%define NUM_BYTES_LIMIT (NUM_WORDS * 4 + 4) +%define SRCDAT4 DWORD [rN + rD * 1] -XOR_NEXT macro +%macro XOR_NEXT 0 mov x1, [rD] xor r0, r1 -endm +%endmacro -else ; NUM_WORDS > 1 +%else ; NUM_WORDS > 1 -src_rN_offset equ 8 +%define src_rN_offset 8 ; + 8 for prefetching next 8-bytes after current iteration -NUM_BYTES_LIMIT equ (NUM_WORDS * 4 + 8) +%define NUM_BYTES_LIMIT (NUM_WORDS * 4 + 8) -XOR_NEXT macro - xor r0, QWORD PTR [rD] ; 64-bit read, can be unaligned -endm +%macro XOR_NEXT 0 + xor r0, QWORD [rD] ; 64-bit read, can be unaligned +%endmacro ; 32-bit or 64-bit -LOAD_SRC_MULT4 macro dest:req, word_index:req - mov dest, [rN + rD * 1 + 4 * (word_index) - src_rN_offset]; -endm - -endif +%macro LOAD_SRC_MULT4 2 ; dest, word_index + mov %1, [rN + rD * 1 + 4 * (%2) - src_rN_offset]; +%endmacro +%endif -MY_PROC @CatStr(XzCrc64UpdateT, %(NUM_WORDS * 4)), 4 +MY_PROC AddNum(XzCrc64UpdateT, NUM_WORDS * 4), 4 MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 mov r0, REG_ABI_PARAM_0 ; r0 <- r1 / r7 mov rD, REG_ABI_PARAM_1 ; r11 <- r2 / r6 mov rN, REG_ABI_PARAM_2 ; r10 <- r8 / r2 -if (IS_LINUX gt 0) +%if ABI == LINUX mov rT, REG_ABI_PARAM_3 ; r9 <- r9 / r1 -endif +%endif cmp rN, NUM_BYTES_LIMIT + ALIGN_MASK - jb crc_end + jb .crc_end @@: test rD, ALIGN_MASK jz @F @@ -109,10 +112,10 @@ endif sub rD, rN add rN, src_rN_offset -align 16 +ALIGN 16 @@: -if NUM_WORDS eq 1 +%if NUM_WORDS == 1 mov x1, x0 shr x1, 8 @@ -128,28 +131,33 @@ if NUM_WORDS eq 1 CRC_XOR r0, x2, 1 CRC_XOR r0, x1, 0 -else ; NUM_WORDS > 1 - -if NUM_WORDS ne 2 - k = 2 - while k lt NUM_WORDS +%else ; NUM_WORDS > 1 + +%if NUM_WORDS != 2 + %assign k 2 + + %rep NUM_WORDS + %if k == NUM_WORDS + %exitrep + %endif LOAD_SRC_MULT4 x1, k - crc_op1 textequ + %define crc_op1 xor - if k eq 2 - if (NUM_WORDS and 1) + %if k == 2 + %if (NUM_WORDS & 1) LOAD_SRC_MULT4 x7, NUM_WORDS ; aligned 32-bit LOAD_SRC_MULT4 x6, NUM_WORDS + 1 ; aligned 32-bit shl r6, 32 - else + %else LOAD_SRC_MULT4 r6, NUM_WORDS ; aligned 64-bit - crc_op1 textequ - endif - endif - table = 4 * (NUM_WORDS - 1 - k) + %undef crc_op1 + %define crc_op1 mov + %endif + %endif + %assign table (4 * (NUM_WORDS - 1 - k)) MOVZXLO x3, x1 - CRC_OP crc_op1, r7, x3, 3 + table + CRC_OP crc_op1, r7, x3, 3 + table MOVZXHI x3, x1 shr x1, 16 CRC_XOR r6, x3, 2 + table @@ -157,14 +165,14 @@ if NUM_WORDS ne 2 shr x1, 8 CRC_XOR r7, x3, 1 + table CRC_XOR r6, x1, 0 + table - k = k + 1 - endm - crc_op2 textequ + %assign k k+1 + %endrep + %define crc_op2 xor -else ; NUM_WORDS == 2 - LOAD_SRC_MULT4 r6, NUM_WORDS ; aligned 64-bit - crc_op2 textequ -endif ; NUM_WORDS == 2 +%else ; NUM_WORDS == 2 + LOAD_SRC_MULT4 r6, NUM_WORDS ; aligned 64-bit + %define crc_op2 mov +%endif ; NUM_WORDS == 2 MOVZXHI x3, x0 MOVZXLO x2, x0 @@ -189,7 +197,7 @@ endif ; NUM_WORDS == 2 xor r0, r6 xor r0, r7 -endif ; NUM_WORDS > 1 +%endif ; NUM_WORDS > 1 add rD, NUM_WORDS * 4 jnc @B @@ -199,78 +207,73 @@ endif ; NUM_WORDS > 1 add rN, NUM_BYTES_LIMIT - 1 sub rN, rD -crc_end: +.crc_end: test rN, rN - jz func_end + jz .func_end @@: CRC1b - jnz @B -func_end: + jnz @B +.func_end: MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 -MY_ENDP + MY_ENDP -else +%else ; ================================================================== ; x86 (32-bit) -rD equ r7 -rN equ r1 -rT equ r5 +%define rD r7 +%define rN r1 +%define rT r5 -xA equ x6 -xA_R equ r6 +%define xA x6 +%define xA_R r6 -ifdef x64 - num_VAR equ r8 -else +%if XBITS == 64 + %define num_VAR r8 +%else + %define crc_OFFS (REG_SIZE * 5) -crc_OFFS equ (REG_SIZE * 5) - -if (IS_CDECL gt 0) or (IS_LINUX gt 0) + %if (IS_CDECL == 1) || (ABI == LINUX) ; cdecl or (GNU fastcall) stack: ; (UInt32 *) table ; size_t size ; void * data ; (UInt64) crc ; ret-ip <-(r4) - data_OFFS equ (8 + crc_OFFS) - size_OFFS equ (REG_SIZE + data_OFFS) - table_OFFS equ (REG_SIZE + size_OFFS) - num_VAR equ [r4 + size_OFFS] - table_VAR equ [r4 + table_OFFS] -else + %define data_OFFS (8 + crc_OFFS) + %define size_OFFS (REG_SIZE + data_OFFS) + %define table_OFFS (REG_SIZE + size_OFFS) + %define num_VAR [r4 + size_OFFS] + %define table_VAR [r4 + table_OFFS] + %else ; Windows fastcall: ; r1 = data, r2 = size ; stack: ; (UInt32 *) table ; (UInt64) crc ; ret-ip <-(r4) - table_OFFS equ (8 + crc_OFFS) - table_VAR equ [r4 + table_OFFS] - num_VAR equ table_VAR -endif -endif ; x64 - -SRCDAT4 equ DWORD PTR [rN + rD * 1] + %define table_OFFS (8 + crc_OFFS) + %define table_VAR [r4 + table_OFFS] + %define num_VAR table_VAR + %endif +%endif -CRC_1 macro op:req, dest:req, src:req, t:req, word_index:req - op dest, DWORD PTR [rT + @CatStr(src, _R) * 8 + 0800h * (t) + (word_index) * 4] -endm +%define SRCDAT4 DWORD [rN + rD * 1] -CRC macro op0:req, op1:req, dest0:req, dest1:req, src:req, t:req - CRC_1 op0, dest0, src, t, 0 - CRC_1 op1, dest1, src, t, 1 -endm +%macro CRC 6 ; op0, op1, dest0, dest1, src, t + CRC_1 %1, %3, %5, %6, 0 + CRC_1 %2, %4, %5, %6, 1 +%endmacro -CRC_XOR macro dest0:req, dest1:req, src:req, t:req - CRC xor, xor, dest0, dest1, src, t -endm +%macro CRC_XOR 4 ; dest0, dest1, src, t + CRC xor, xor, %1, %2, %3, %4 +%endmacro -CRC1b macro - movzx xA, BYTE PTR [rD] +%macro CRC1b 0 + movzx xA, BYTE [rD] inc rD MOVZXLO x3, x0 xor xA, x3 @@ -278,12 +281,12 @@ CRC1b macro shr x2, 8 CRC_XOR x0, x2, xA, 0 dec rN -endm +%endmacro -MY_PROLOG_BASE macro +%macro MY_PROLOG_BASE 0 MY_PUSH_4_REGS -ifdef x64 + %if XBITS == 64 mov r0, REG_ABI_PARAM_0 ; r0 <- r1 / r7 mov rT, REG_ABI_PARAM_3 ; r5 <- r9 / r1 mov rN, REG_ABI_PARAM_2 ; r1 <- r8 / r2 @@ -291,55 +294,52 @@ ifdef x64 mov r2, r0 shr r2, 32 mov x0, x0 -else - if (IS_CDECL gt 0) or (IS_LINUX gt 0) - proc_numParams = proc_numParams + 2 ; for ABI_LINUX + %else + %if (IS_CDECL == 1) || (ABI == LINUX) + %assign proc_numParams proc_numParams + 2 ; for ABI_LINUX mov rN, [r4 + size_OFFS] mov rD, [r4 + data_OFFS] - else + %else mov rD, REG_ABI_PARAM_0 ; r7 <- r1 : (data) mov rN, REG_ABI_PARAM_1 ; r1 <- r2 : (size) - endif + %endif mov x0, [r4 + crc_OFFS] mov x2, [r4 + crc_OFFS + 4] mov rT, table_VAR -endif -endm + %endif +%endmacro -MY_EPILOG_BASE macro crc_end:req, func_end:req -crc_end: +%macro MY_EPILOG_BASE 0 +.crc_end: test rN, rN - jz func_end + jz %%func_end @@: CRC1b - jnz @B -func_end: -ifdef x64 + jnz @B +%%func_end: + %if XBITS == 64 shl r2, 32 xor r0, r2 -endif + %endif MY_POP_4_REGS -endm +%endmacro -; ALIGN_MASK is 3 or 7 bytes alignment: -ALIGN_MASK equ (7 - (NUM_WORDS and 1) * 4) - -if (NUM_WORDS eq 1) +%if NUM_WORDS == 1 -NUM_BYTES_LIMIT_T4 equ (NUM_WORDS * 4 + 4) +%define NUM_BYTES_LIMIT_T4 (NUM_WORDS * 4 + 4) -MY_PROC @CatStr(XzCrc64UpdateT, %(NUM_WORDS * 4)), 5 +MY_PROC AddNum(XzCrc64UpdateT, NUM_WORDS * 4), 5 MY_PROLOG_BASE cmp rN, NUM_BYTES_LIMIT_T4 + ALIGN_MASK - jb crc_end_4 + jb .crc_end @@: test rD, ALIGN_MASK - jz @F + jz @B CRC1b - jmp @B + jmp @F @@: xor x0, [rD] lea rN, [rD + rN * 1 - (NUM_BYTES_LIMIT_T4 - 1)] @@ -347,7 +347,7 @@ MY_PROC @CatStr(XzCrc64UpdateT, %(NUM_WORDS * 4)), 5 add rN, 4 MOVZXLO xA, x0 -align 16 +ALIGN 16 @@: mov x3, SRCDAT4 xor x3, x2 @@ -374,108 +374,105 @@ align 16 xor x0, [rD] add rN, NUM_BYTES_LIMIT_T4 - 1 sub rN, rD - MY_EPILOG_BASE crc_end_4, func_end_4 -MY_ENDP - -else ; NUM_WORDS > 1 - -SHR_X macro x, imm - shr x, imm -endm - - -ITER_1 macro v0, v1, a, off - MOVZXLO xA, a - SHR_X a, 8 - CRC_XOR v0, v1, xA, off -endm - - -ITER_4 macro v0, v1, a, off -if 0 eq 0 - ITER_1 v0, v1, a, off + 3 - ITER_1 v0, v1, a, off + 2 - ITER_1 v0, v1, a, off + 1 - CRC_XOR v0, v1, a, off -elseif 0 eq 0 - MOVZXLO xA, a - CRC_XOR v0, v1, xA, off + 3 - mov xA, a - ror a, 16 ; 32-bit ror + + MY_EPILOG_BASE + MY_ENDP + +%else ; NUM_WORDS > 1 + + +%macro ITER_1 4 + MOVZXLO xA, %3 + shr %3, 8 + CRC_XOR %1, %2, xA, %4 +%endmacro + + +%macro ITER_4 4 ; v0, v1, a, off + %if 0 == 0 + ITER_1 %1, %2, %3, %4 + 3 + ITER_1 %1, %2, %3, %4 + 2 + ITER_1 %1, %2, %3, %4 + 1 + CRC_XOR %1, %2, %3, %4 + %elif 0 == 0 + MOVZXLO xA, %3 + CRC_XOR %1, %2, xA, %4 + 3 + mov xA, %3 + ror %3, 16 ; 32-bit ror shr xA, 24 - CRC_XOR v0, v1, xA, off - MOVZXLO xA, a - SHR_X a, 24 - CRC_XOR v0, v1, xA, off + 1 - CRC_XOR v0, v1, a, off + 2 -else + CRC_XOR %1, %2, xA, %4 + movzx xA, %3 + shr %3, 24 + CRC_XOR %1, %2, xA, %4 + 1 + CRC_XOR %1, %2, %3, %4 + 2 + %else ; MOVZXHI provides smaller code, but MOVZX_HI_BYTE is not fast instruction - MOVZXLO xA, a - CRC_XOR v0, v1, xA, off + 3 - MOVZXHI xA, a - SHR_X a, 16 - CRC_XOR v0, v1, xA, off + 2 - MOVZXLO xA, a - SHR_X a, 8 - CRC_XOR v0, v1, xA, off + 1 - CRC_XOR v0, v1, a, off -endif -endm - - - -ITER_1_PAIR macro v0, v1, a0, a1, off - ITER_1 v0, v1, a0, off + 4 - ITER_1 v0, v1, a1, off -endm - -src_rD_offset equ 8 -STEP_SIZE equ (NUM_WORDS * 4) - -ITER_12_NEXT macro op, index, v0, v1 - op v0, DWORD PTR [rD + (index + 1) * STEP_SIZE - src_rD_offset] - op v1, DWORD PTR [rD + (index + 1) * STEP_SIZE + 4 - src_rD_offset] -endm - -ITER_12 macro index, a0, a1, v0, v1 - - if NUM_SKIP_BYTES eq 0 - ITER_12_NEXT mov, index, v0, v1 - else - k = 0 - while k lt NUM_SKIP_BYTES - movzx xA, BYTE PTR [rD + (index) * STEP_SIZE + k + 8 - src_rD_offset] - if k eq 0 - CRC mov, mov, v0, v1, xA, NUM_SKIP_BYTES - 1 - k - else - CRC_XOR v0, v1, xA, NUM_SKIP_BYTES - 1 - k - endif - k = k + 1 - endm - ITER_12_NEXT xor, index, v0, v1 - endif - -if 0 eq 0 - ITER_4 v0, v1, a0, NUM_SKIP_BYTES + 4 - ITER_4 v0, v1, a1, NUM_SKIP_BYTES -else ; interleave version is faster/slower for different processors - ITER_1_PAIR v0, v1, a0, a1, NUM_SKIP_BYTES + 3 - ITER_1_PAIR v0, v1, a0, a1, NUM_SKIP_BYTES + 2 - ITER_1_PAIR v0, v1, a0, a1, NUM_SKIP_BYTES + 1 - CRC_XOR v0, v1, a0, NUM_SKIP_BYTES + 4 - CRC_XOR v0, v1, a1, NUM_SKIP_BYTES -endif -endm + MOVZXLO xA, %3 + CRC_XOR %1, %2, xA, %4 + 3 + MOVZXHI xA, %3 + shr %3, 16 + CRC_XOR %1, %2, xA, %4 + 2 + MOVZXLO xA, %3 + shr %3, 8 + CRC_XOR %1, %2, xA, %4 + 1 + CRC_XOR %1, %2, %3, %4 + %endif +%endmacro + + +%macro ITER_1_PAIR 5 ; v0, v1, a0, a1, off + ITER_1 %1, %2, %3, %5 + 4 + ITER_1 %1, %2, %4, %5 +%endmacro + +%define src_rD_offset 8 +%define STEP_SIZE (NUM_WORDS * 4) + +%macro ITER_12_NEXT 4 ; op, index, v0, v1 + %1 %3, DWORD [rD + (%2 + 1) * STEP_SIZE - src_rD_offset] + %1 %4, DWORD [rD + (%2 + 1) * STEP_SIZE + 4 - src_rD_offset] +%endmacro + +%macro ITER_12 5 + %assign index %1 + + %if NUM_SKIP_BYTES == 0 + ITER_12_NEXT mov, index, %4, %5 + %else + %assign k 0 + %rep NUM_SKIP_BYTES + movzx xA, BYTE [rD + (index) * STEP_SIZE + k + 8 - src_rD_offset] + %if k == 0 + CRC mov, mov, %4, %5, xA, NUM_SKIP_BYTES - 1 - k + %else + CRC_XOR %4, %5, xA, NUM_SKIP_BYTES - 1 - k + %endif + %assign k k+1 + %endrep + ITER_12_NEXT xor, index, %4, %5 + %endif + + %if 0 == 0 + ITER_4 %4, %5, %2, NUM_SKIP_BYTES + 4 + ITER_4 %4, %5, %3, NUM_SKIP_BYTES + %else ; interleave version is faster/slower for different processors + ITER_1_PAIR %4, %5, %2, %3, NUM_SKIP_BYTES + 3 + ITER_1_PAIR %4, %5, %2, %3, NUM_SKIP_BYTES + 2 + ITER_1_PAIR %4, %5, %2, %3, NUM_SKIP_BYTES + 1 + CRC_XOR %4, %5, %2, NUM_SKIP_BYTES + 4 + CRC_XOR %4, %5, %3, NUM_SKIP_BYTES + %endif +%endmacro ; we use (UNROLL_CNT > 1) to reduce read ports pressure (num_VAR reads) -UNROLL_CNT equ (2 * 1) -NUM_BYTES_LIMIT equ (STEP_SIZE * UNROLL_CNT + 8) +%define UNROLL_CNT (2 * 1) +%define NUM_BYTES_LIMIT (STEP_SIZE * UNROLL_CNT + 8) -MY_PROC @CatStr(XzCrc64UpdateT, %(NUM_WORDS * 4)), 5 +MY_PROC AddNum(XzCrc64UpdateT, NUM_WORDS * 4), 5 MY_PROLOG_BASE cmp rN, NUM_BYTES_LIMIT + ALIGN_MASK - jb crc_end_12 + jb .crc_end @@: test rD, ALIGN_MASK jz @F @@ -488,22 +485,22 @@ MY_PROC @CatStr(XzCrc64UpdateT, %(NUM_WORDS * 4)), 5 lea rN, [rD + rN * 1 - (NUM_BYTES_LIMIT - 1)] mov num_VAR, rN -align 16 +ALIGN 16 @@: - i = 0 - rept UNROLL_CNT - if (i and 1) eq 0 - ITER_12 i, x0, x2, x1, x3 - else - ITER_12 i, x1, x3, x0, x2 - endif - i = i + 1 - endm - - if (UNROLL_CNT and 1) + %assign i 0 + %rep UNROLL_CNT + %if (i & 1) == 0 + ITER_12 i, x0, x2, x1, x3 + %else + ITER_12 i, x1, x3, x0, x2 + %endif + %assign i i+1 + %endrep + + %if (UNROLL_CNT & 1) mov x0, x1 mov x2, x3 - endif + %endif add rD, STEP_SIZE * UNROLL_CNT cmp rD, num_VAR jb @B @@ -515,9 +512,9 @@ align 16 xor x0, [rD] xor x2, [rD + 4] - MY_EPILOG_BASE crc_end_12, func_end_12 -MY_ENDP + MY_EPILOG_BASE + MY_ENDP + +%endif ; (NUM_WORDS > 1) +%endif ; ! x64 -endif ; (NUM_WORDS > 1) -endif ; ! x64 -end diff --git a/C/7zip_gcc_c.mak b/C/7zip_gcc_c.mak index 006cfe069..7085aa484 100644 --- a/C/7zip_gcc_c.mak +++ b/C/7zip_gcc_c.mak @@ -1,8 +1,7 @@ MY_ARCH_2 = $(MY_ARCH) -MY_ASM = jwasm -MY_ASM = asmc +MY_ASM = nasm ifndef RC #RC=windres.exe --target=pe-x86-64 @@ -129,14 +128,12 @@ endif ifdef IS_X64 -AFLAGS_ABI = -elf64 -DABI_LINUX +AFLAGS_ABI = -felf64 else -AFLAGS_ABI = -elf -DABI_LINUX -DABI_CDECL -# -DABI_CDECL -# -DABI_LINUX -# -DABI_CDECL +AFLAGS_ABI = -felf32 endif -AFLAGS = $(AFLAGS_ABI) -Fo$(O)/ + +AFLAGS = $(AFLAGS_ABI) -I$(/dev/null | grep 'x86_64'), ) +IS_X64 = 1 +endif + +USE_ASM = 1 + ifdef USE_ASM ifdef IS_X64 USE_LZMA_DEC_ASM=1 diff --git a/CPP/Build.mak b/CPP/Build.mak index d9fcfb52f..196cee39e 100644 --- a/CPP/Build.mak +++ b/CPP/Build.mak @@ -26,14 +26,16 @@ O=o !IF "$(PLATFORM)" == "x64" -MY_ML = ml64 -WX +MY_ML = nasm -fwin64 +#MY_ML = ml64 -WX #-Dx64 !ELSEIF "$(PLATFORM)" == "arm64" MY_ML = armasm64 !ELSEIF "$(PLATFORM)" == "arm" MY_ML = armasm -WX !ELSE -MY_ML = ml -WX +MY_ML = nasm -fwin32 +#MY_ML = ml -WX # -DABI_CDECL !ENDIF @@ -60,7 +62,8 @@ COMPL_ASM = $(MY_ML) $** $O/$(*B).obj !ELSEIF "$(PLATFORM)" == "arm64" COMPL_ASM = $(MY_ML) $** $O/$(*B).obj !ELSE -COMPL_ASM = $(MY_ML) -c -Fo$O/ $** +#COMPL_ASM = $(MY_ML) -c -Fo$O/ $** +COMPL_ASM = $(MY_ML) -I$(*D) -o $O/$(*B).obj $** !ENDIF !IFDEF OLD_COMPILER From 2c17aff1e3809e783004503017fe4b4f7f0b5875 Mon Sep 17 00:00:00 2001 From: darealshinji Date: Mon, 3 Aug 2026 17:25:40 +0200 Subject: [PATCH 2/2] XzCrc64Opt: jump labels and register names --- Asm/x86/XzCrc64Opt.asm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Asm/x86/XzCrc64Opt.asm b/Asm/x86/XzCrc64Opt.asm index a1095a4c8..cdb096bd2 100644 --- a/Asm/x86/XzCrc64Opt.asm +++ b/Asm/x86/XzCrc64Opt.asm @@ -23,8 +23,8 @@ MY_ASM_START %if n == 0 %fatal %else - ; x ==> r - %1 %2, [rT + %tok(%strcat('r', %eval(n-1))) * 8 + 0800h * (%4) + (%5) * 4] + ; x ==> x_R + %1 %2, [rT + %tok(%strcat('x', %eval(n-1), "_R")) * 8 + 0800h * (%4) + (%5) * 4] %endif %endmacro @@ -337,9 +337,9 @@ MY_PROC AddNum(XzCrc64UpdateT, NUM_WORDS * 4), 5 jb .crc_end @@: test rD, ALIGN_MASK - jz @B + jz @F CRC1b - jmp @F + jmp @B @@: xor x0, [rD] lea rN, [rD + rN * 1 - (NUM_BYTES_LIMIT_T4 - 1)]