Commit d5d325ea authored by David S. Miller's avatar David S. Miller

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Alexei Starovoitov says:

====================
pull-request: bpf 2020-09-15

The following pull-request contains BPF updates for your *net* tree.

We've added 12 non-merge commits during the last 19 day(s) which contain
a total of 10 files changed, 47 insertions(+), 38 deletions(-).

The main changes are:

1) docs/bpf fixes, from Andrii.

2) ld_abs fix, from Daniel.

3) socket casting helpers fix, from Martin.

4) hash iterator fixes, from Yonghong.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 2fbc6e89 ce880cb8
...@@ -182,9 +182,6 @@ in the order of reservations, but only after all previous records where ...@@ -182,9 +182,6 @@ in the order of reservations, but only after all previous records where
already committed. It is thus possible for slow producers to temporarily hold already committed. It is thus possible for slow producers to temporarily hold
off submitted records, that were reserved later. off submitted records, that were reserved later.
Reservation/commit/consumer protocol is verified by litmus tests in
Documentation/litmus_tests/bpf-rb/_.
One interesting implementation bit, that significantly simplifies (and thus One interesting implementation bit, that significantly simplifies (and thus
speeds up as well) implementation of both producers and consumers is how data speeds up as well) implementation of both producers and consumers is how data
area is mapped twice contiguously back-to-back in the virtual memory. This area is mapped twice contiguously back-to-back in the virtual memory. This
...@@ -200,7 +197,7 @@ a self-pacing notifications of new data being availability. ...@@ -200,7 +197,7 @@ a self-pacing notifications of new data being availability.
being available after commit only if consumer has already caught up right up to being available after commit only if consumer has already caught up right up to
the record being committed. If not, consumer still has to catch up and thus the record being committed. If not, consumer still has to catch up and thus
will see new data anyways without needing an extra poll notification. will see new data anyways without needing an extra poll notification.
Benchmarks (see tools/testing/selftests/bpf/benchs/bench_ringbuf.c_) show that Benchmarks (see tools/testing/selftests/bpf/benchs/bench_ringbufs.c) show that
this allows to achieve a very high throughput without having to resort to this allows to achieve a very high throughput without having to resort to
tricks like "notify only every Nth sample", which are necessary with perf tricks like "notify only every Nth sample", which are necessary with perf
buffer. For extreme cases, when BPF program wants more manual control of buffer. For extreme cases, when BPF program wants more manual control of
......
...@@ -1622,7 +1622,6 @@ struct bpf_iter_seq_hash_map_info { ...@@ -1622,7 +1622,6 @@ struct bpf_iter_seq_hash_map_info {
struct bpf_map *map; struct bpf_map *map;
struct bpf_htab *htab; struct bpf_htab *htab;
void *percpu_value_buf; // non-zero means percpu hash void *percpu_value_buf; // non-zero means percpu hash
unsigned long flags;
u32 bucket_id; u32 bucket_id;
u32 skip_elems; u32 skip_elems;
}; };
...@@ -1632,7 +1631,6 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info, ...@@ -1632,7 +1631,6 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info,
struct htab_elem *prev_elem) struct htab_elem *prev_elem)
{ {
const struct bpf_htab *htab = info->htab; const struct bpf_htab *htab = info->htab;
unsigned long flags = info->flags;
u32 skip_elems = info->skip_elems; u32 skip_elems = info->skip_elems;
u32 bucket_id = info->bucket_id; u32 bucket_id = info->bucket_id;
struct hlist_nulls_head *head; struct hlist_nulls_head *head;
...@@ -1656,19 +1654,18 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info, ...@@ -1656,19 +1654,18 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info,
/* not found, unlock and go to the next bucket */ /* not found, unlock and go to the next bucket */
b = &htab->buckets[bucket_id++]; b = &htab->buckets[bucket_id++];
htab_unlock_bucket(htab, b, flags); rcu_read_unlock();
skip_elems = 0; skip_elems = 0;
} }
for (i = bucket_id; i < htab->n_buckets; i++) { for (i = bucket_id; i < htab->n_buckets; i++) {
b = &htab->buckets[i]; b = &htab->buckets[i];
flags = htab_lock_bucket(htab, b); rcu_read_lock();
count = 0; count = 0;
head = &b->head; head = &b->head;
hlist_nulls_for_each_entry_rcu(elem, n, head, hash_node) { hlist_nulls_for_each_entry_rcu(elem, n, head, hash_node) {
if (count >= skip_elems) { if (count >= skip_elems) {
info->flags = flags;
info->bucket_id = i; info->bucket_id = i;
info->skip_elems = count; info->skip_elems = count;
return elem; return elem;
...@@ -1676,7 +1673,7 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info, ...@@ -1676,7 +1673,7 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info,
count++; count++;
} }
htab_unlock_bucket(htab, b, flags); rcu_read_unlock();
skip_elems = 0; skip_elems = 0;
} }
...@@ -1754,14 +1751,10 @@ static int bpf_hash_map_seq_show(struct seq_file *seq, void *v) ...@@ -1754,14 +1751,10 @@ static int bpf_hash_map_seq_show(struct seq_file *seq, void *v)
static void bpf_hash_map_seq_stop(struct seq_file *seq, void *v) static void bpf_hash_map_seq_stop(struct seq_file *seq, void *v)
{ {
struct bpf_iter_seq_hash_map_info *info = seq->private;
if (!v) if (!v)
(void)__bpf_hash_map_seq_show(seq, NULL); (void)__bpf_hash_map_seq_show(seq, NULL);
else else
htab_unlock_bucket(info->htab, rcu_read_unlock();
&info->htab->buckets[info->bucket_id],
info->flags);
} }
static int bpf_iter_init_hash_map(void *priv_data, static int bpf_iter_init_hash_map(void *priv_data,
......
...@@ -226,10 +226,12 @@ static void *map_seq_next(struct seq_file *m, void *v, loff_t *pos) ...@@ -226,10 +226,12 @@ static void *map_seq_next(struct seq_file *m, void *v, loff_t *pos)
else else
prev_key = key; prev_key = key;
rcu_read_lock();
if (map->ops->map_get_next_key(map, prev_key, key)) { if (map->ops->map_get_next_key(map, prev_key, key)) {
map_iter(m)->done = true; map_iter(m)->done = true;
return NULL; key = NULL;
} }
rcu_read_unlock();
return key; return key;
} }
......
...@@ -7066,8 +7066,6 @@ static int bpf_gen_ld_abs(const struct bpf_insn *orig, ...@@ -7066,8 +7066,6 @@ static int bpf_gen_ld_abs(const struct bpf_insn *orig,
bool indirect = BPF_MODE(orig->code) == BPF_IND; bool indirect = BPF_MODE(orig->code) == BPF_IND;
struct bpf_insn *insn = insn_buf; struct bpf_insn *insn = insn_buf;
/* We're guaranteed here that CTX is in R6. */
*insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_CTX);
if (!indirect) { if (!indirect) {
*insn++ = BPF_MOV64_IMM(BPF_REG_2, orig->imm); *insn++ = BPF_MOV64_IMM(BPF_REG_2, orig->imm);
} else { } else {
...@@ -7075,6 +7073,8 @@ static int bpf_gen_ld_abs(const struct bpf_insn *orig, ...@@ -7075,6 +7073,8 @@ static int bpf_gen_ld_abs(const struct bpf_insn *orig,
if (orig->imm) if (orig->imm)
*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, orig->imm); *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, orig->imm);
} }
/* We're guaranteed here that CTX is in R6. */
*insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_CTX);
switch (BPF_SIZE(orig->code)) { switch (BPF_SIZE(orig->code)) {
case BPF_B: case BPF_B:
...@@ -9523,7 +9523,7 @@ BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk) ...@@ -9523,7 +9523,7 @@ BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk)
* trigger an explicit type generation here. * trigger an explicit type generation here.
*/ */
BTF_TYPE_EMIT(struct tcp6_sock); BTF_TYPE_EMIT(struct tcp6_sock);
if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP && if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP &&
sk->sk_family == AF_INET6) sk->sk_family == AF_INET6)
return (unsigned long)sk; return (unsigned long)sk;
...@@ -9541,7 +9541,7 @@ const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto = { ...@@ -9541,7 +9541,7 @@ const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto = {
BPF_CALL_1(bpf_skc_to_tcp_sock, struct sock *, sk) BPF_CALL_1(bpf_skc_to_tcp_sock, struct sock *, sk)
{ {
if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP) if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
return (unsigned long)sk; return (unsigned long)sk;
return (unsigned long)NULL; return (unsigned long)NULL;
...@@ -9559,12 +9559,12 @@ const struct bpf_func_proto bpf_skc_to_tcp_sock_proto = { ...@@ -9559,12 +9559,12 @@ const struct bpf_func_proto bpf_skc_to_tcp_sock_proto = {
BPF_CALL_1(bpf_skc_to_tcp_timewait_sock, struct sock *, sk) BPF_CALL_1(bpf_skc_to_tcp_timewait_sock, struct sock *, sk)
{ {
#ifdef CONFIG_INET #ifdef CONFIG_INET
if (sk->sk_prot == &tcp_prot && sk->sk_state == TCP_TIME_WAIT) if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_TIME_WAIT)
return (unsigned long)sk; return (unsigned long)sk;
#endif #endif
#if IS_BUILTIN(CONFIG_IPV6) #if IS_BUILTIN(CONFIG_IPV6)
if (sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_TIME_WAIT) if (sk && sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_TIME_WAIT)
return (unsigned long)sk; return (unsigned long)sk;
#endif #endif
...@@ -9583,12 +9583,12 @@ const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto = { ...@@ -9583,12 +9583,12 @@ const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto = {
BPF_CALL_1(bpf_skc_to_tcp_request_sock, struct sock *, sk) BPF_CALL_1(bpf_skc_to_tcp_request_sock, struct sock *, sk)
{ {
#ifdef CONFIG_INET #ifdef CONFIG_INET
if (sk->sk_prot == &tcp_prot && sk->sk_state == TCP_NEW_SYN_RECV) if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_NEW_SYN_RECV)
return (unsigned long)sk; return (unsigned long)sk;
#endif #endif
#if IS_BUILTIN(CONFIG_IPV6) #if IS_BUILTIN(CONFIG_IPV6)
if (sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_NEW_SYN_RECV) if (sk && sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_NEW_SYN_RECV)
return (unsigned long)sk; return (unsigned long)sk;
#endif #endif
...@@ -9610,7 +9610,7 @@ BPF_CALL_1(bpf_skc_to_udp6_sock, struct sock *, sk) ...@@ -9610,7 +9610,7 @@ BPF_CALL_1(bpf_skc_to_udp6_sock, struct sock *, sk)
* trigger an explicit type generation here. * trigger an explicit type generation here.
*/ */
BTF_TYPE_EMIT(struct udp6_sock); BTF_TYPE_EMIT(struct udp6_sock);
if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_UDP && if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_UDP &&
sk->sk_type == SOCK_DGRAM && sk->sk_family == AF_INET6) sk->sk_type == SOCK_DGRAM && sk->sk_family == AF_INET6)
return (unsigned long)sk; return (unsigned long)sk;
......
...@@ -303,10 +303,10 @@ static int xdp_umem_account_pages(struct xdp_umem *umem) ...@@ -303,10 +303,10 @@ static int xdp_umem_account_pages(struct xdp_umem *umem)
static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
{ {
u32 npgs_rem, chunk_size = mr->chunk_size, headroom = mr->headroom;
bool unaligned_chunks = mr->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG; bool unaligned_chunks = mr->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG;
u32 chunk_size = mr->chunk_size, headroom = mr->headroom;
u64 npgs, addr = mr->addr, size = mr->len; u64 npgs, addr = mr->addr, size = mr->len;
unsigned int chunks, chunks_per_page; unsigned int chunks, chunks_rem;
int err; int err;
if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) { if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) {
...@@ -336,19 +336,18 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) ...@@ -336,19 +336,18 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
if ((addr + size) < addr) if ((addr + size) < addr)
return -EINVAL; return -EINVAL;
npgs = size >> PAGE_SHIFT; npgs = div_u64_rem(size, PAGE_SIZE, &npgs_rem);
if (npgs_rem)
npgs++;
if (npgs > U32_MAX) if (npgs > U32_MAX)
return -EINVAL; return -EINVAL;
chunks = (unsigned int)div_u64(size, chunk_size); chunks = (unsigned int)div_u64_rem(size, chunk_size, &chunks_rem);
if (chunks == 0) if (chunks == 0)
return -EINVAL; return -EINVAL;
if (!unaligned_chunks) { if (!unaligned_chunks && chunks_rem)
chunks_per_page = PAGE_SIZE / chunk_size;
if (chunks < chunks_per_page || chunks % chunks_per_page)
return -EINVAL; return -EINVAL;
}
if (headroom >= chunk_size - XDP_PACKET_HEADROOM) if (headroom >= chunk_size - XDP_PACKET_HEADROOM)
return -EINVAL; return -EINVAL;
......
...@@ -38,7 +38,7 @@ FEATURE_TESTS = libbfd disassembler-four-args ...@@ -38,7 +38,7 @@ FEATURE_TESTS = libbfd disassembler-four-args
FEATURE_DISPLAY = libbfd disassembler-four-args FEATURE_DISPLAY = libbfd disassembler-four-args
check_feat := 1 check_feat := 1
NON_CHECK_FEAT_TARGETS := clean bpftool_clean runqslower_clean NON_CHECK_FEAT_TARGETS := clean bpftool_clean runqslower_clean resolve_btfids_clean
ifdef MAKECMDGOALS ifdef MAKECMDGOALS
ifeq ($(filter-out $(NON_CHECK_FEAT_TARGETS),$(MAKECMDGOALS)),) ifeq ($(filter-out $(NON_CHECK_FEAT_TARGETS),$(MAKECMDGOALS)),)
check_feat := 0 check_feat := 0
...@@ -89,7 +89,7 @@ $(OUTPUT)bpf_exp.lex.c: $(OUTPUT)bpf_exp.yacc.c ...@@ -89,7 +89,7 @@ $(OUTPUT)bpf_exp.lex.c: $(OUTPUT)bpf_exp.yacc.c
$(OUTPUT)bpf_exp.yacc.o: $(OUTPUT)bpf_exp.yacc.c $(OUTPUT)bpf_exp.yacc.o: $(OUTPUT)bpf_exp.yacc.c
$(OUTPUT)bpf_exp.lex.o: $(OUTPUT)bpf_exp.lex.c $(OUTPUT)bpf_exp.lex.o: $(OUTPUT)bpf_exp.lex.c
clean: bpftool_clean runqslower_clean clean: bpftool_clean runqslower_clean resolve_btfids_clean
$(call QUIET_CLEAN, bpf-progs) $(call QUIET_CLEAN, bpf-progs)
$(Q)$(RM) -r -- $(OUTPUT)*.o $(OUTPUT)bpf_jit_disasm $(OUTPUT)bpf_dbg \ $(Q)$(RM) -r -- $(OUTPUT)*.o $(OUTPUT)bpf_jit_disasm $(OUTPUT)bpf_dbg \
$(OUTPUT)bpf_asm $(OUTPUT)bpf_exp.yacc.* $(OUTPUT)bpf_exp.lex.* $(OUTPUT)bpf_asm $(OUTPUT)bpf_exp.yacc.* $(OUTPUT)bpf_exp.lex.*
......
...@@ -80,6 +80,7 @@ libbpf-clean: ...@@ -80,6 +80,7 @@ libbpf-clean:
clean: libsubcmd-clean libbpf-clean fixdep-clean clean: libsubcmd-clean libbpf-clean fixdep-clean
$(call msg,CLEAN,$(BINARY)) $(call msg,CLEAN,$(BINARY))
$(Q)$(RM) -f $(BINARY); \ $(Q)$(RM) -f $(BINARY); \
$(RM) -rf $(if $(OUTPUT),$(OUTPUT),.)/feature; \
find $(if $(OUTPUT),$(OUTPUT),.) -name \*.o -or -name \*.o.cmd -or -name \*.o.d | xargs $(RM) find $(if $(OUTPUT),$(OUTPUT),.) -name \*.o -or -name \*.o.cmd -or -name \*.o.d | xargs $(RM)
tags: tags:
......
...@@ -59,7 +59,7 @@ FEATURE_USER = .libbpf ...@@ -59,7 +59,7 @@ FEATURE_USER = .libbpf
FEATURE_TESTS = libelf libelf-mmap zlib bpf reallocarray FEATURE_TESTS = libelf libelf-mmap zlib bpf reallocarray
FEATURE_DISPLAY = libelf zlib bpf FEATURE_DISPLAY = libelf zlib bpf
INCLUDES = -I. -I$(srctree)/tools/include -I$(srctree)/tools/arch/$(ARCH)/include/uapi -I$(srctree)/tools/include/uapi INCLUDES = -I. -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi
FEATURE_CHECK_CFLAGS-bpf = $(INCLUDES) FEATURE_CHECK_CFLAGS-bpf = $(INCLUDES)
check_feat := 1 check_feat := 1
...@@ -152,6 +152,7 @@ GLOBAL_SYM_COUNT = $(shell readelf -s --wide $(BPF_IN_SHARED) | \ ...@@ -152,6 +152,7 @@ GLOBAL_SYM_COUNT = $(shell readelf -s --wide $(BPF_IN_SHARED) | \
awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}' | \ awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}' | \
sort -u | wc -l) sort -u | wc -l)
VERSIONED_SYM_COUNT = $(shell readelf --dyn-syms --wide $(OUTPUT)libbpf.so | \ VERSIONED_SYM_COUNT = $(shell readelf --dyn-syms --wide $(OUTPUT)libbpf.so | \
awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}' | \
grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | sort -u | wc -l) grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | sort -u | wc -l)
CMD_TARGETS = $(LIB_TARGET) $(PC_FILE) CMD_TARGETS = $(LIB_TARGET) $(PC_FILE)
...@@ -219,6 +220,7 @@ check_abi: $(OUTPUT)libbpf.so ...@@ -219,6 +220,7 @@ check_abi: $(OUTPUT)libbpf.so
awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}'| \ awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}'| \
sort -u > $(OUTPUT)libbpf_global_syms.tmp; \ sort -u > $(OUTPUT)libbpf_global_syms.tmp; \
readelf --dyn-syms --wide $(OUTPUT)libbpf.so | \ readelf --dyn-syms --wide $(OUTPUT)libbpf.so | \
awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}'| \
grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | \ grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | \
sort -u > $(OUTPUT)libbpf_versioned_syms.tmp; \ sort -u > $(OUTPUT)libbpf_versioned_syms.tmp; \
diff -u $(OUTPUT)libbpf_global_syms.tmp \ diff -u $(OUTPUT)libbpf_global_syms.tmp \
......
...@@ -5203,8 +5203,8 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, ...@@ -5203,8 +5203,8 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj,
int i, j, nrels, new_sz; int i, j, nrels, new_sz;
const struct btf_var_secinfo *vi = NULL; const struct btf_var_secinfo *vi = NULL;
const struct btf_type *sec, *var, *def; const struct btf_type *sec, *var, *def;
struct bpf_map *map = NULL, *targ_map;
const struct btf_member *member; const struct btf_member *member;
struct bpf_map *map, *targ_map;
const char *name, *mname; const char *name, *mname;
Elf_Data *symbols; Elf_Data *symbols;
unsigned int moff; unsigned int moff;
......
...@@ -47,7 +47,10 @@ int dump_bpf_hash_map(struct bpf_iter__bpf_map_elem *ctx) ...@@ -47,7 +47,10 @@ int dump_bpf_hash_map(struct bpf_iter__bpf_map_elem *ctx)
__u32 seq_num = ctx->meta->seq_num; __u32 seq_num = ctx->meta->seq_num;
struct bpf_map *map = ctx->map; struct bpf_map *map = ctx->map;
struct key_t *key = ctx->key; struct key_t *key = ctx->key;
struct key_t tmp_key;
__u64 *val = ctx->value; __u64 *val = ctx->value;
__u64 tmp_val = 0;
int ret;
if (in_test_mode) { if (in_test_mode) {
/* test mode is used by selftests to /* test mode is used by selftests to
...@@ -61,6 +64,18 @@ int dump_bpf_hash_map(struct bpf_iter__bpf_map_elem *ctx) ...@@ -61,6 +64,18 @@ int dump_bpf_hash_map(struct bpf_iter__bpf_map_elem *ctx)
if (key == (void *)0 || val == (void *)0) if (key == (void *)0 || val == (void *)0)
return 0; return 0;
/* update the value and then delete the <key, value> pair.
* it should not impact the existing 'val' which is still
* accessible under rcu.
*/
__builtin_memcpy(&tmp_key, key, sizeof(struct key_t));
ret = bpf_map_update_elem(&hashmap1, &tmp_key, &tmp_val, 0);
if (ret)
return 0;
ret = bpf_map_delete_elem(&hashmap1, &tmp_key);
if (ret)
return 0;
key_sum_a += key->a; key_sum_a += key->a;
key_sum_b += key->b; key_sum_b += key->b;
key_sum_c += key->c; key_sum_c += key->c;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment