• Peter Zijlstra's avatar
    sched/clock, x86: Use mul_u64_u32_shr() for native_sched_clock() · 5dd12c21
    Peter Zijlstra authored
    Use mul_u64_u32_shr() so that x86_64 can use a single 64x64->128 mul.
    
    Before:
    
    0000000000000560 <native_sched_clock>:
     560:   44 8b 1d 00 00 00 00    mov    0x0(%rip),%r11d        # 567 <native_sched_clock+0x7>
     567:   55                      push   %rbp
     568:   48 89 e5                mov    %rsp,%rbp
     56b:   45 85 db                test   %r11d,%r11d
     56e:   75 4f                   jne    5bf <native_sched_clock+0x5f>
     570:   0f 31                   rdtsc
     572:   89 c0                   mov    %eax,%eax
     574:   48 c1 e2 20             shl    $0x20,%rdx
     578:   48 c7 c1 00 00 00 00    mov    $0x0,%rcx
     57f:   48 09 c2                or     %rax,%rdx
     582:   48 c7 c7 00 00 00 00    mov    $0x0,%rdi
     589:   65 8b 04 25 00 00 00    mov    %gs:0x0,%eax
     590:   00
     591:   48 98                   cltq
     593:   48 8b 34 c5 00 00 00    mov    0x0(,%rax,8),%rsi
     59a:   00
     59b:   48 89 d0                mov    %rdx,%rax
     59e:   81 e2 ff 03 00 00       and    $0x3ff,%edx
     5a4:   48 c1 e8 0a             shr    $0xa,%rax
     5a8:   48 0f af 14 0e          imul   (%rsi,%rcx,1),%rdx
     5ad:   48 0f af 04 0e          imul   (%rsi,%rcx,1),%rax
     5b2:   5d                      pop    %rbp
     5b3:   48 03 04 3e             add    (%rsi,%rdi,1),%rax
     5b7:   48 c1 ea 0a             shr    $0xa,%rdx
     5bb:   48 01 d0                add    %rdx,%rax
     5be:   c3                      retq
    
    After:
    
    0000000000000550 <native_sched_clock>:
     550:   8b 3d 00 00 00 00       mov    0x0(%rip),%edi        # 556 <native_sched_clock+0x6>
     556:   55                      push   %rbp
     557:   48 89 e5                mov    %rsp,%rbp
     55a:   48 83 e4 f0             and    $0xfffffffffffffff0,%rsp
     55e:   85 ff                   test   %edi,%edi
     560:   75 2c                   jne    58e <native_sched_clock+0x3e>
     562:   0f 31                   rdtsc
     564:   89 c0                   mov    %eax,%eax
     566:   48 c1 e2 20             shl    $0x20,%rdx
     56a:   48 09 c2                or     %rax,%rdx
     56d:   65 48 8b 04 25 00 00    mov    %gs:0x0,%rax
     574:   00 00
     576:   89 c0                   mov    %eax,%eax
     578:   48 f7 e2                mul    %rdx
     57b:   65 48 8b 0c 25 00 00    mov    %gs:0x0,%rcx
     582:   00 00
     584:   c9                      leaveq
     585:   48 0f ac d0 0a          shrd   $0xa,%rdx,%rax
     58a:   48 01 c8                add    %rcx,%rax
     58d:   c3                      retq
    
                            MAINLINE   POST
    
        sched_clock_stable: 1	   1
        (cold) sched_clock: 329841     331312
        (cold) local_clock: 301773     310296
        (warm) sched_clock: 38375      38247
        (warm) local_clock: 100371     102713
        (warm) rdtsc:       27340      27289
        sched_clock_stable: 0          0
        (cold) sched_clock: 382634     372706
        (cold) local_clock: 396890     399275
        (warm) sched_clock: 38194      38124
        (warm) local_clock: 143452     148698
        (warm) rdtsc:       27345      27365
    Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
    Cc: Linus Torvalds <torvalds@linux-foundation.org>
    Cc: Andrew Morton <akpm@linux-foundation.org>
    Link: http://lkml.kernel.org/n/tip-piu203ses5y1g36bnyw2n16x@git.kernel.orgSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
    5dd12c21
timer.h 2.11 KB