Commit 9ac589cf authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Herbert Xu

crypto: x86/crc32 - Use local .L symbols for code

Avoid cluttering up the kallsyms symbol table with entries that should
not end up in things like backtraces, as they have undescriptive and
generated identifiers.
Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 1d4b0ff3
...@@ -90,7 +90,7 @@ SYM_FUNC_START(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligne ...@@ -90,7 +90,7 @@ SYM_FUNC_START(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligne
sub $0x40, LEN sub $0x40, LEN
add $0x40, BUF add $0x40, BUF
cmp $0x40, LEN cmp $0x40, LEN
jb less_64 jb .Lless_64
#ifdef __x86_64__ #ifdef __x86_64__
movdqa .Lconstant_R2R1(%rip), CONSTANT movdqa .Lconstant_R2R1(%rip), CONSTANT
...@@ -98,7 +98,7 @@ SYM_FUNC_START(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligne ...@@ -98,7 +98,7 @@ SYM_FUNC_START(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligne
movdqa .Lconstant_R2R1, CONSTANT movdqa .Lconstant_R2R1, CONSTANT
#endif #endif
loop_64:/* 64 bytes Full cache line folding */ .Lloop_64:/* 64 bytes Full cache line folding */
prefetchnta 0x40(BUF) prefetchnta 0x40(BUF)
movdqa %xmm1, %xmm5 movdqa %xmm1, %xmm5
movdqa %xmm2, %xmm6 movdqa %xmm2, %xmm6
...@@ -139,8 +139,8 @@ loop_64:/* 64 bytes Full cache line folding */ ...@@ -139,8 +139,8 @@ loop_64:/* 64 bytes Full cache line folding */
sub $0x40, LEN sub $0x40, LEN
add $0x40, BUF add $0x40, BUF
cmp $0x40, LEN cmp $0x40, LEN
jge loop_64 jge .Lloop_64
less_64:/* Folding cache line into 128bit */ .Lless_64:/* Folding cache line into 128bit */
#ifdef __x86_64__ #ifdef __x86_64__
movdqa .Lconstant_R4R3(%rip), CONSTANT movdqa .Lconstant_R4R3(%rip), CONSTANT
#else #else
...@@ -167,8 +167,8 @@ less_64:/* Folding cache line into 128bit */ ...@@ -167,8 +167,8 @@ less_64:/* Folding cache line into 128bit */
pxor %xmm4, %xmm1 pxor %xmm4, %xmm1
cmp $0x10, LEN cmp $0x10, LEN
jb fold_64 jb .Lfold_64
loop_16:/* Folding rest buffer into 128bit */ .Lloop_16:/* Folding rest buffer into 128bit */
movdqa %xmm1, %xmm5 movdqa %xmm1, %xmm5
pclmulqdq $0x00, CONSTANT, %xmm1 pclmulqdq $0x00, CONSTANT, %xmm1
pclmulqdq $0x11, CONSTANT, %xmm5 pclmulqdq $0x11, CONSTANT, %xmm5
...@@ -177,9 +177,9 @@ loop_16:/* Folding rest buffer into 128bit */ ...@@ -177,9 +177,9 @@ loop_16:/* Folding rest buffer into 128bit */
sub $0x10, LEN sub $0x10, LEN
add $0x10, BUF add $0x10, BUF
cmp $0x10, LEN cmp $0x10, LEN
jge loop_16 jge .Lloop_16
fold_64: .Lfold_64:
/* perform the last 64 bit fold, also adds 32 zeroes /* perform the last 64 bit fold, also adds 32 zeroes
* to the input stream */ * to the input stream */
pclmulqdq $0x01, %xmm1, CONSTANT /* R4 * xmm1.low */ pclmulqdq $0x01, %xmm1, CONSTANT /* R4 * xmm1.low */
......
...@@ -49,15 +49,15 @@ ...@@ -49,15 +49,15 @@
## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
.macro LABEL prefix n .macro LABEL prefix n
\prefix\n\(): .L\prefix\n\():
.endm .endm
.macro JMPTBL_ENTRY i .macro JMPTBL_ENTRY i
.quad crc_\i .quad .Lcrc_\i
.endm .endm
.macro JNC_LESS_THAN j .macro JNC_LESS_THAN j
jnc less_than_\j jnc .Lless_than_\j
.endm .endm
# Define threshold where buffers are considered "small" and routed to more # Define threshold where buffers are considered "small" and routed to more
...@@ -108,30 +108,30 @@ SYM_FUNC_START(crc_pcl) ...@@ -108,30 +108,30 @@ SYM_FUNC_START(crc_pcl)
neg %bufp neg %bufp
and $7, %bufp # calculate the unalignment amount of and $7, %bufp # calculate the unalignment amount of
# the address # the address
je proc_block # Skip if aligned je .Lproc_block # Skip if aligned
## If len is less than 8 and we're unaligned, we need to jump ## If len is less than 8 and we're unaligned, we need to jump
## to special code to avoid reading beyond the end of the buffer ## to special code to avoid reading beyond the end of the buffer
cmp $8, len cmp $8, len
jae do_align jae .Ldo_align
# less_than_8 expects length in upper 3 bits of len_dw # less_than_8 expects length in upper 3 bits of len_dw
# less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30] # less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30]
shl $32-3+1, len_dw shl $32-3+1, len_dw
jmp less_than_8_post_shl1 jmp .Lless_than_8_post_shl1
do_align: .Ldo_align:
#### Calculate CRC of unaligned bytes of the buffer (if any) #### Calculate CRC of unaligned bytes of the buffer (if any)
movq (bufptmp), tmp # load a quadward from the buffer movq (bufptmp), tmp # load a quadward from the buffer
add %bufp, bufptmp # align buffer pointer for quadword add %bufp, bufptmp # align buffer pointer for quadword
# processing # processing
sub %bufp, len # update buffer length sub %bufp, len # update buffer length
align_loop: .Lalign_loop:
crc32b %bl, crc_init_dw # compute crc32 of 1-byte crc32b %bl, crc_init_dw # compute crc32 of 1-byte
shr $8, tmp # get next byte shr $8, tmp # get next byte
dec %bufp dec %bufp
jne align_loop jne .Lalign_loop
proc_block: .Lproc_block:
################################################################ ################################################################
## 2) PROCESS BLOCKS: ## 2) PROCESS BLOCKS:
...@@ -141,11 +141,11 @@ proc_block: ...@@ -141,11 +141,11 @@ proc_block:
movq len, tmp # save num bytes in tmp movq len, tmp # save num bytes in tmp
cmpq $128*24, len cmpq $128*24, len
jae full_block jae .Lfull_block
continue_block: .Lcontinue_block:
cmpq $SMALL_SIZE, len cmpq $SMALL_SIZE, len
jb small jb .Lsmall
## len < 128*24 ## len < 128*24
movq $2731, %rax # 2731 = ceil(2^16 / 24) movq $2731, %rax # 2731 = ceil(2^16 / 24)
...@@ -175,7 +175,7 @@ continue_block: ...@@ -175,7 +175,7 @@ continue_block:
################################################################ ################################################################
## 2a) PROCESS FULL BLOCKS: ## 2a) PROCESS FULL BLOCKS:
################################################################ ################################################################
full_block: .Lfull_block:
movl $128,%eax movl $128,%eax
lea 128*8*2(block_0), block_1 lea 128*8*2(block_0), block_1
lea 128*8*3(block_0), block_2 lea 128*8*3(block_0), block_2
...@@ -190,7 +190,6 @@ full_block: ...@@ -190,7 +190,6 @@ full_block:
## 3) CRC Array: ## 3) CRC Array:
################################################################ ################################################################
crc_array:
i=128 i=128
.rept 128-1 .rept 128-1
.altmacro .altmacro
...@@ -243,28 +242,28 @@ LABEL crc_ 0 ...@@ -243,28 +242,28 @@ LABEL crc_ 0
ENDBR ENDBR
mov tmp, len mov tmp, len
cmp $128*24, tmp cmp $128*24, tmp
jae full_block jae .Lfull_block
cmp $24, tmp cmp $24, tmp
jae continue_block jae .Lcontinue_block
less_than_24: .Lless_than_24:
shl $32-4, len_dw # less_than_16 expects length shl $32-4, len_dw # less_than_16 expects length
# in upper 4 bits of len_dw # in upper 4 bits of len_dw
jnc less_than_16 jnc .Lless_than_16
crc32q (bufptmp), crc_init crc32q (bufptmp), crc_init
crc32q 8(bufptmp), crc_init crc32q 8(bufptmp), crc_init
jz do_return jz .Ldo_return
add $16, bufptmp add $16, bufptmp
# len is less than 8 if we got here # len is less than 8 if we got here
# less_than_8 expects length in upper 3 bits of len_dw # less_than_8 expects length in upper 3 bits of len_dw
# less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30] # less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30]
shl $2, len_dw shl $2, len_dw
jmp less_than_8_post_shl1 jmp .Lless_than_8_post_shl1
####################################################################### #######################################################################
## 6) LESS THAN 256-bytes REMAIN AT THIS POINT (8-bits of len are full) ## 6) LESS THAN 256-bytes REMAIN AT THIS POINT (8-bits of len are full)
####################################################################### #######################################################################
small: .Lsmall:
shl $32-8, len_dw # Prepare len_dw for less_than_256 shl $32-8, len_dw # Prepare len_dw for less_than_256
j=256 j=256
.rept 5 # j = {256, 128, 64, 32, 16} .rept 5 # j = {256, 128, 64, 32, 16}
...@@ -280,32 +279,32 @@ LABEL less_than_ %j # less_than_j: Length should be in ...@@ -280,32 +279,32 @@ LABEL less_than_ %j # less_than_j: Length should be in
crc32q i(bufptmp), crc_init # Compute crc32 of 8-byte data crc32q i(bufptmp), crc_init # Compute crc32 of 8-byte data
i=i+8 i=i+8
.endr .endr
jz do_return # Return if remaining length is zero jz .Ldo_return # Return if remaining length is zero
add $j, bufptmp # Advance buf add $j, bufptmp # Advance buf
.endr .endr
less_than_8: # Length should be stored in .Lless_than_8: # Length should be stored in
# upper 3 bits of len_dw # upper 3 bits of len_dw
shl $1, len_dw shl $1, len_dw
less_than_8_post_shl1: .Lless_than_8_post_shl1:
jnc less_than_4 jnc .Lless_than_4
crc32l (bufptmp), crc_init_dw # CRC of 4 bytes crc32l (bufptmp), crc_init_dw # CRC of 4 bytes
jz do_return # return if remaining data is zero jz .Ldo_return # return if remaining data is zero
add $4, bufptmp add $4, bufptmp
less_than_4: # Length should be stored in .Lless_than_4: # Length should be stored in
# upper 2 bits of len_dw # upper 2 bits of len_dw
shl $1, len_dw shl $1, len_dw
jnc less_than_2 jnc .Lless_than_2
crc32w (bufptmp), crc_init_dw # CRC of 2 bytes crc32w (bufptmp), crc_init_dw # CRC of 2 bytes
jz do_return # return if remaining data is zero jz .Ldo_return # return if remaining data is zero
add $2, bufptmp add $2, bufptmp
less_than_2: # Length should be stored in the MSB .Lless_than_2: # Length should be stored in the MSB
# of len_dw # of len_dw
shl $1, len_dw shl $1, len_dw
jnc less_than_1 jnc .Lless_than_1
crc32b (bufptmp), crc_init_dw # CRC of 1 byte crc32b (bufptmp), crc_init_dw # CRC of 1 byte
less_than_1: # Length should be zero .Lless_than_1: # Length should be zero
do_return: .Ldo_return:
movq crc_init, %rax movq crc_init, %rax
popq %rsi popq %rsi
popq %rdi popq %rdi
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment