osmocom-net-gprs

osmocom-net-gprs@lists.osmocom.org

483 discussions

[PATCH net 3/4] gtp: fix an use-after-free in ipv4_pdp_find()
by Taehee Yoo 11 Dec '19

11 Dec '19

ipv4_pdp_find() is called in TX packet path of GTP. ipv4_pdp_find() internally uses gtp->tid_hash to lookup pdp context. In the current code, gtp->tid_hash and gtp->addr_hash are freed by ->dellink(), which is gtp_dellink(). But gtp_dellink() would be called while packets are processing. So, gtp_dellink() should not free gtp->tid_hash and gtp->addr_hash. Instead, dev->priv_destructor() would be used because this callback is called after all packet processing safely. Test commands: ip link add veth1 type veth peer name veth2 ip a a 172.0.0.1/24 dev veth1 ip link set veth1 up ip a a 172.99.0.1/32 dev lo gtp-link add gtp1 & gtp-tunnel add gtp1 v1 200 100 172.99.0.2 172.0.0.2 ip r a 172.99.0.2/32 dev gtp1 ip link set gtp1 mtu 1500 ip netns add ns2 ip link set veth2 netns ns2 ip netns exec ns2 ip a a 172.0.0.2/24 dev veth2 ip netns exec ns2 ip link set veth2 up ip netns exec ns2 ip a a 172.99.0.2/32 dev lo ip netns exec ns2 ip link set lo up ip netns exec ns2 gtp-link add gtp2 & ip netns exec ns2 gtp-tunnel add gtp2 v1 100 200 172.99.0.1 172.0.0.1 ip netns exec ns2 ip r a 172.99.0.1/32 dev gtp2 ip netns exec ns2 ip link set gtp2 mtu 1500 hping3 172.99.0.2 -2 --flood & ip link del gtp1 Splat looks like: [ 72.568081][ T1195] BUG: KASAN: use-after-free in ipv4_pdp_find.isra.12+0x130/0x170 [gtp] [ 72.568916][ T1195] Read of size 8 at addr ffff8880b9a35d28 by task hping3/1195 [ 72.569631][ T1195] [ 72.569861][ T1195] CPU: 2 PID: 1195 Comm: hping3 Not tainted 5.5.0-rc1 #199 [ 72.570547][ T1195] Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006 [ 72.571438][ T1195] Call Trace: [ 72.571764][ T1195] dump_stack+0x96/0xdb [ 72.572171][ T1195] ? ipv4_pdp_find.isra.12+0x130/0x170 [gtp] [ 72.572761][ T1195] print_address_description.constprop.5+0x1be/0x360 [ 72.573400][ T1195] ? ipv4_pdp_find.isra.12+0x130/0x170 [gtp] [ 72.573971][ T1195] ? ipv4_pdp_find.isra.12+0x130/0x170 [gtp] [ 72.574544][ T1195] __kasan_report+0x12a/0x16f [ 72.575014][ T1195] ? ipv4_pdp_find.isra.12+0x130/0x170 [gtp] [ 72.575593][ T1195] kasan_report+0xe/0x20 [ 72.576004][ T1195] ipv4_pdp_find.isra.12+0x130/0x170 [gtp] [ 72.576577][ T1195] gtp_build_skb_ip4+0x199/0x1420 [gtp] [ ... ] [ 72.647671][ T1195] BUG: unable to handle page fault for address: ffff8880b9a35d28 [ 72.648512][ T1195] #PF: supervisor read access in kernel mode [ 72.649158][ T1195] #PF: error_code(0x0000) - not-present page [ 72.649849][ T1195] PGD a6c01067 P4D a6c01067 PUD 11fb07067 PMD 11f939067 PTE 800fffff465ca060 [ 72.652958][ T1195] Oops: 0000 [#1] SMP DEBUG_PAGEALLOC KASAN PTI [ 72.653834][ T1195] CPU: 2 PID: 1195 Comm: hping3 Tainted: G B 5.5.0-rc1 #199 [ 72.668062][ T1195] RIP: 0010:ipv4_pdp_find.isra.12+0x86/0x170 [gtp] [ ... ] [ 72.679168][ T1195] Call Trace: [ 72.679603][ T1195] gtp_build_skb_ip4+0x199/0x1420 [gtp] [ 72.681915][ T1195] ? ipv4_pdp_find.isra.12+0x170/0x170 [gtp] [ 72.682513][ T1195] ? lock_acquire+0x164/0x3b0 [ 72.682966][ T1195] ? gtp_dev_xmit+0x35e/0x890 [gtp] [ 72.683481][ T1195] gtp_dev_xmit+0x3c2/0x890 [gtp] [ ... ] Fixes: 459aa660eb1d ("gtp: add initial driver for datapath of GPRS Tunneling Protocol (GTP-U)") Signed-off-by: Taehee Yoo <ap420073(a)gmail.com> --- drivers/net/gtp.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index a010e0a11c33..5450b1099c6d 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -640,9 +640,16 @@ static void gtp_link_setup(struct net_device *dev) } static int gtp_hashtable_new(struct gtp_dev *gtp, int hsize); -static void gtp_hashtable_free(struct gtp_dev *gtp); static int gtp_encap_enable(struct gtp_dev *gtp, struct nlattr *data[]); +static void gtp_destructor(struct net_device *dev) +{ + struct gtp_dev *gtp = netdev_priv(dev); + + kfree(gtp->addr_hash); + kfree(gtp->tid_hash); +} + static int gtp_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack) @@ -677,13 +684,15 @@ static int gtp_newlink(struct net *src_net, struct net_device *dev, gn = net_generic(dev_net(dev), gtp_net_id); list_add_rcu(&gtp->list, &gn->gtp_dev_list); + dev->priv_destructor = gtp_destructor; netdev_dbg(dev, "registered new GTP interface\n"); return 0; out_hashtable: - gtp_hashtable_free(gtp); + kfree(gtp->addr_hash); + kfree(gtp->tid_hash); out_encap: gtp_encap_disable(gtp); return err; @@ -692,8 +701,13 @@ static int gtp_newlink(struct net *src_net, struct net_device *dev, static void gtp_dellink(struct net_device *dev, struct list_head *head) { struct gtp_dev *gtp = netdev_priv(dev); + struct pdp_ctx *pctx; + int i; + + for (i = 0; i < gtp->hash_size; i++) + hlist_for_each_entry_rcu(pctx, &gtp->tid_hash[i], hlist_tid) + pdp_context_delete(pctx); - gtp_hashtable_free(gtp); list_del_rcu(&gtp->list); unregister_netdevice_queue(dev, head); } @@ -771,20 +785,6 @@ static int gtp_hashtable_new(struct gtp_dev *gtp, int hsize) return -ENOMEM; } -static void gtp_hashtable_free(struct gtp_dev *gtp) -{ - struct pdp_ctx *pctx; - int i; - - for (i = 0; i < gtp->hash_size; i++) - hlist_for_each_entry_rcu(pctx, &gtp->tid_hash[i], hlist_tid) - pdp_context_delete(pctx); - - synchronize_rcu(); - kfree(gtp->addr_hash); - kfree(gtp->tid_hash); -} - static struct sock *gtp_encap_enable_socket(int fd, int type, struct gtp_dev *gtp) { -- 2.17.1

1 0

[PATCH net 2/4] gtp: fix wrong condition in gtp_genl_dump_pdp()
by Taehee Yoo 11 Dec '19

11 Dec '19

gtp_genl_dump_pdp() is ->dumpit() callback of GTP module and it is used to dump pdp contexts. it would be re-executed because of dump packet size. If dump packet size is too big, it saves current dump pointer (gtp interface pointer, bucket, TID value) then it restarts dump from last pointer. Current GTP code allows adding zero TID pdp context but dump code ignores zero TID value. So, last dump pointer will not be found. In addition, this patch adds missing rcu_read_lock() in gtp_genl_dump_pdp(). Fixes: 459aa660eb1d ("gtp: add initial driver for datapath of GPRS Tunneling Protocol (GTP-U)") Signed-off-by: Taehee Yoo <ap420073(a)gmail.com> --- drivers/net/gtp.c | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index 8b742edf793d..a010e0a11c33 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -38,7 +38,6 @@ struct pdp_ctx { struct hlist_node hlist_addr; union { - u64 tid; struct { u64 tid; u16 flow; @@ -1244,43 +1243,46 @@ static int gtp_genl_dump_pdp(struct sk_buff *skb, struct netlink_callback *cb) { struct gtp_dev *last_gtp = (struct gtp_dev *)cb->args[2], *gtp; + int i, j, bucket = cb->args[0], skip = cb->args[1]; struct net *net = sock_net(skb->sk); - struct gtp_net *gn = net_generic(net, gtp_net_id); - unsigned long tid = cb->args[1]; - int i, k = cb->args[0], ret; struct pdp_ctx *pctx; + struct gtp_net *gn; + + gn = net_generic(net, gtp_net_id); if (cb->args[4]) return 0; + rcu_read_lock(); list_for_each_entry_rcu(gtp, &gn->gtp_dev_list, list) { if (last_gtp && last_gtp != gtp) continue; else last_gtp = NULL; - for (i = k; i < gtp->hash_size; i++) { - hlist_for_each_entry_rcu(pctx, &gtp->tid_hash[i], hlist_tid) { - if (tid && tid != pctx->u.tid) - continue; - else - tid = 0; - - ret = gtp_genl_fill_info(skb, - NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, - cb->nlh->nlmsg_type, pctx); - if (ret < 0) { + for (i = bucket; i < gtp->hash_size; i++) { + j = 0; + hlist_for_each_entry_rcu(pctx, &gtp->tid_hash[i], + hlist_tid) { + if (j >= skip && + gtp_genl_fill_info(skb, + NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, + cb->nlh->nlmsg_type, pctx)) { cb->args[0] = i; - cb->args[1] = pctx->u.tid; + cb->args[1] = j; cb->args[2] = (unsigned long)gtp; goto out; } + j++; } + skip = 0; } + bucket = 0; } cb->args[4] = 1; out: + rcu_read_unlock(); return skb->len; } -- 2.17.1

1 0

[PATCH net 1/4] gtp: do not allow adding duplicate tid and ms_addr pdp context
by Taehee Yoo 11 Dec '19

11 Dec '19

GTP RX packet path lookups pdp context with TID. If duplicate TID pdp contexts are existing in the list, it couldn't select correct pdp context. So, TID value should be unique. GTP TX packet path lookups pdp context with ms_addr. If duplicate ms_addr pdp contexts are existing in the list, it couldn't select correct pdp context. So, ms_addr value should be unique. Fixes: 459aa660eb1d ("gtp: add initial driver for datapath of GPRS Tunneling Protocol (GTP-U)") Signed-off-by: Taehee Yoo <ap420073(a)gmail.com> --- drivers/net/gtp.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index ecfe26215935..8b742edf793d 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -926,24 +926,31 @@ static void ipv4_pdp_fill(struct pdp_ctx *pctx, struct genl_info *info) } } -static int ipv4_pdp_add(struct gtp_dev *gtp, struct sock *sk, - struct genl_info *info) +static int gtp_pdp_add(struct gtp_dev *gtp, struct sock *sk, + struct genl_info *info) { + struct pdp_ctx *pctx, *pctx_tid = NULL; struct net_device *dev = gtp->dev; u32 hash_ms, hash_tid = 0; - struct pdp_ctx *pctx; + unsigned int version; bool found = false; __be32 ms_addr; ms_addr = nla_get_be32(info->attrs[GTPA_MS_ADDRESS]); hash_ms = ipv4_hashfn(ms_addr) % gtp->hash_size; + version = nla_get_u32(info->attrs[GTPA_VERSION]); - hlist_for_each_entry_rcu(pctx, &gtp->addr_hash[hash_ms], hlist_addr) { - if (pctx->ms_addr_ip4.s_addr == ms_addr) { - found = true; - break; - } - } + pctx = ipv4_pdp_find(gtp, ms_addr); + if (pctx) + found = true; + if (version == GTP_V0) + pctx_tid = gtp0_pdp_find(gtp, + nla_get_u64(info->attrs[GTPA_TID])); + else if (version == GTP_V1) + pctx_tid = gtp1_pdp_find(gtp, + nla_get_u32(info->attrs[GTPA_I_TEI])); + if (pctx_tid) + found = true; if (found) { if (info->nlhdr->nlmsg_flags & NLM_F_EXCL) @@ -951,6 +958,11 @@ static int ipv4_pdp_add(struct gtp_dev *gtp, struct sock *sk, if (info->nlhdr->nlmsg_flags & NLM_F_REPLACE) return -EOPNOTSUPP; + if (pctx && pctx_tid) + return -EEXIST; + if (!pctx) + pctx = pctx_tid; + ipv4_pdp_fill(pctx, info); if (pctx->gtp_version == GTP_V0) @@ -1074,7 +1086,7 @@ static int gtp_genl_new_pdp(struct sk_buff *skb, struct genl_info *info) goto out_unlock; } - err = ipv4_pdp_add(gtp, sk, info); + err = gtp_pdp_add(gtp, sk, info); out_unlock: rcu_read_unlock(); -- 2.17.1

1 0

OsmoDevCon 2020 registration
by Harald Welte 02 Dec '19

02 Dec '19

Dear fellow Osmocom developers, I would like to invite all developers and contributors to Osmocom [sub]projects to register for OsmoDevCon 2020 (held on April 24th-27th, 2020 in Berlin). For details known so far, please check http://osmocom.org/projects/osmo-dev-con/wiki/OsmoDevCon2020 Please enter your name at https://osmocom.org/projects/osmo-dev-con/wiki/OsmoDevCon2020#Requested in case you would like to attend. Registering early allows proper planning. Thanks! Looking forward to meeting old and new Osmocom developers in April 2020. Regards, Harald -- - Harald Welte <laforge(a)osmocom.org> http://laforge.gnumonks.org/ ============================================================================ "Privacy in residential applications is a desirable marketing option." (ETSI EN 300 175-7 Ch. A6)

1 0

Re: high ksoftirqd while using module gtp?
by Tony Clark 22 Nov '19

22 Nov '19

Firstly I would like to say great thanks Firat for the reply, it certainly put me on a different investigation path. And apologies for not replying sooner. I wanted to make sure it was the correct path before I replied back to the group with the findings and associated solution. If the GTP-U connection is connecting to the PG-W with a single IP at (src/dst) each side, and UDP flow has isn't been enabled on the network card of the host using gtp.ko in the kernel all the associated network traffic will be received on a single queue on the network, which is then serviced by a single ksoftirq thread. at somepoint the system will be receiving more traffic than there is available thread space to service the request, your ksoftirq will burn at 100%. That means all your traffic will be bound to a single network queue, bound to a single irq thread, and limit your overall throughput, no matter how big your network pipe is. This is because the network card hashed the packet via SRC_IP:SRC_PORT:DEST_IP:DEST_PORT:PROTO to a single queue. # take note on the discussions about udp-flow-hash udp4 using ethtool https://home.regit.org/tag/performance/ https://www.joyent.com/blog/virtualizing-nics https://www.serializing.me/2015/04/25/rxtx-buffers-rss-others-on-boot/ You can check if your card supports adjustable parameters by using "ethtool -k DEV | egrep -v fixed". As firat eludes to (below) udp flow hashing should be supported. If you enabled UDP flow hash then it will spread the hash over multiple queues. The default number of queues on the network card can vary, depending on your hardware firmware driver, and any additional associated kernel parameters. Would recommend having the latest firmware driver for your network card, and latest kernel driver for the network card if possible. Alas the network cards used by my hardware didn't support flow hash, it had intel flow director, which wasn't granular enough and worked with TCP, so to work around this limitation having multiple SRC_IPs in different name spaces with the same GTP UDP PORT numbers resolved the problem. Of course if you are sending GTP-U to a single destination from multiple sources (say 6 IP's), via 6 different kernel name spaces, you spread the load over 6 queues, which is better than nothing on a limited feature network card. Time to upgrade the 10G network card.... This took the system from 100% ksoftirq on a single cpu running at throughput 1G, to around 7 to 8GIG throughput at 90% ksoftirq over multiple cpu;s... There is still massive room for improvement. For performance some things to investigate/consider... Which I had different levels of success... Here are my ramblings..... on the linux host... Assuming your traffic is now spread across multiple queues (above) - or at least spread as best as can be... Kernel sysctl tweaking is always of benefit, if your using out of the box kernel config... Example udp buffers, queue sizes, paging and virtual memory settings... There is a application called "tuned", which allows you to adjust profiles for the kernel sysctl... My performance profile which suited the testing best was "throuhput-performance" if your looking for straight performance, disable audit processing like "auditd". Question use of SELINUX, enforcing/permissive or disabled. can bring results on performance, if you doing testing or load testing... ofcourse its a security consideration.. If you don't need to use ipfilters/firewall in my case can increase the throughput by a 3rd by disabling (cleaning the filter tables and unloading the modules). Black listing the modules so they dont get loaded at kernel time. Note you can stop modules getting loaded with kernel.modules_disabled=1, but be careful if your also messing with initramfs rebuilds, because you don't get any modules once your set that parameter, i learnt the hard way :) Investigate smp_affinity and affinity_hint, along with irqbalane using the --hintpolicy=exact. understand which irq's service the network cards, and how many queues you have.. /proc/interrupts will guide you (grep 'CPU|rxtx' /proc/interrupts)... understand the smp_affinity numbers.. "for ((irq=START_IRQ; irq<=END_IRQ; irq++)); do cat /proc/irq/$irq/smp_affinity; done | sort –u", as you can adjust which queue goes to which ksoftirq to manually balance the queues if you so desire. brilliant document on irq debugging.... https://events.static.linuxfound.org/sites/events/files/slides/LinuxConJapa… you can monitor what calls are been executed on cpu's by using... I found this most useful to understand that ipfilter was eating a significant amount of CPU cycles, and also what other calls are eating up cycles inside ksoftirq. https://github.com/brendangregg/FlameGraph Investigate additional memory management using numactl (numa daemon). remember if you are using virtualisation you might want to pin guests to specific sockets, along with numa pinning on the vmhost.. Also look at reserved memory allocation in the vmhost for the guest... This will make your guest perform better. enable sysstat (sar) as it will aid your investigation if you havent already (sar -u ALL -P ALL 1). This will show which softirqs are eating most cpu and to which cpu they are bound, this also translates directly to the network queue that the traffic is coming in on.. Ie, network card queue 6, talks to cpu/6 talking to irq/6 and so on... Using flamegraph will help you understand what syscalls and chewing the CPU.. If your using virtualisation then the number of default queues that vxnet (vmware in this example) presents to the guest might be less than the number of network card queues the vmhost sees (so watch out for that). You can adjust the number of queues to the guest by params in the vmware network driver... investigate VDMQ / netqueue, to increase the number of available hardware queues from the vmhost to the guest. depending which quest driver your using vxnet3, or others some drivers dont support NAPI (see further down). * VMDQ: array of int* * Number of Virtual Machine Device Queues: 0/1 = disable, 2-16 enable (default=8)* * RSS: array of int* * Number of Receive-Side Scaling Descriptor Queues, default 1=number of cpus* * MQ: array of int* * Disable or enable Multiple Queues, default 1* Node: array of int set the starting node to allocate memory on, default -1 * IntMode: array of int* * Change Interrupt Mode (0=Legacy, 1=MSI, 2=MSI-X), default 2* * InterruptType: array of int* * Change Interrupt Mode (0=Legacy, 1=MSI, 2=MSI-X), default IntMode (deprecated)* Make sure your virtual switch (vmware) if used has Pass-through (Direct-path I/O) enabled. NIC teaming policy should be validated depending on your requirement, example Policy "route based on IP hash" can be of benefit. Check the network card is MSI-X, and the linux driver supports NAPI (most should these days, but you never know), also check your vmhost driver supports napi, if not get a NAPI supported kvm driver, or vmware driver (vib update). Upgrade your kernel, to a later release 4.x.. even consider using a later distro of linux... I tried fedora 29. I also compiled latest osmocom from source, with compile options for "optimisation -O3 and other such". "bmon -b" was a good tool understand throughput loads, along with loading through qdisc/fq_dodel mq's.... Understand qdisc via ip link or ifconfig ( http://tldp.org/HOWTO/Traffic-Control-HOWTO/components.html), adjusting the queues has some traction, but if unsure leave as default. TSO/UFo?GSO/LRO/GRO - understand your network card with respects to these, this can improve performance if you haven't already enabled (or adversely disabled options, since sometimes it doesn't actually help). You can get the your card options using ethool TCP Segmentation Offload (TSO) Uses the TCP protocol to send large packets. Uses the NIC to handle segmentation, and then adds the TCP, IP and data link layer protocol headers to each segment. UDP Fragmentation Offload (UFO) Uses the UDP protocol to send large packets. Uses the NIC to handle IP fragmentation into MTU sized packets for large UDP datagrams. Generic Segmentation Offload (GSO) Uses the TCP or UDP protocol to send large packets. If the NIC cannot handle segmentation/fragmentation, GSO performs the same operations, bypassing the NIC hardware. This is achieved by delaying segmentation until as late as possible, for example, when the packet is processed by the device driver. Large Receive Offload (LRO) Uses the TCP protocol. All incoming packets are re-segmented as they are received, reducing the number of segments the system has to process. They can be merged either in the driver or using the NIC. A problem with LRO is that it tends to resegment all incoming packets, often ignoring differences in headers and other information which can cause errors. It is generally not possible to use LRO when IP forwarding is enabled. LRO in combination with IP forwarding can lead to checksum errors. Forwarding is enabled if /proc/sys/net/ipv4/ip_forward is set to 1. Generic Receive Offload (GRO) Uses either the TCP or UDP protocols. GRO is more rigorous than LRO when resegmenting packets. For example it checks the MAC headers of each packet, which must match, only a limited number of TCP or IP headers can be different, and the TCP timestamps must match. Resegmenting can be handled by either the NIC or the GSO code. Traffic steering was on by default with the version of linux i was using, but worth checking if your using older versions. https://www.kernel.org/doc/Documentation/networking/scaling.txt (from the txt link) note: Some advanced NICs allow steering packets to queues based on programmable filters. For example, webserver bound TCP port 80 packets can be directed to their own receive queue. Such â€œn-tupleâ€ filters can be configured from ethtool (--config-ntuple). Interestingly investigate your network card, for its hashing algorithms, how it distributes the traffic over its ring buffers, you can on some cards adjust the RSS hash function. Alas the card i was using stuck to "toeplitz" for hits hashing, which others were disabled and unavailable / xor and crc32. The indirection table can be adjusted based on the tuplets "ethtool -X" but didn't really assist too much on this. ethtool -x <dev> RX flow hash indirection table for ens192 with 8 RX ring(s): 0: 0 1 2 3 4 5 6 7 8: 0 1 2 3 4 5 6 7 16: 0 1 2 3 4 5 6 7 24: 0 1 2 3 4 5 6 7 RSS hash key: Operation not supported RSS hash function: toeplitz: on xor: off crc32: off Check the default size of the rx/tx ring buffers, they maybe suboptimal. ethtool -g ens192 Ring parameters for ens192: Pre-set maximums: RX: 4096 RX Mini: 0 RX Jumbo: 4096 TX: 4096 Current hardware settings: RX: 1024 RX Mini: 0 RX Jumbo: 256 TX: 512 If your using port channels, make sure you have the correct hashing policy enabled at the switch end... I haven't investigated this option yet but some switches also do scaling, to assist (certainly with virtualisation)... Maybe one day i will get around to this... Additionally CISCO describe that you should have VM-FEX optimisation https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualizati… note: *table 4.* Scaling of Dynamic vNIC with VMDirectPath, Virtual Machines Running on Linux Guest with VMXNET3 Emulated Driver and Multi-Queue Enabled *Table 5.* Scaling of Dynamic vNIC with VMDirectPath, Virtual Machines Running on Linux Guest with VMXNET3 Emulated Driver and Multi-Queue Disabled Another thing to consider/investigate - openvswitch/bridging... If your using eth pairs to send your traffic down name spaces... you can have some varied results with performance by trying openvswitch/brctl I really enjoyed the investigation path, again thanks to Firat for the pointer, otherwise it would have taken longer to get the answer... Tony On Fri, Jun 21, 2019 at 6:50 AM fırat sönmez <firatssonmez(a)gmail.com> wrote: > Hi, > > It has been over 2 years that I have worked with gtp and I kind of had the > same problem that time, we had a 10gbit cable and tried to see how much udp > flow we could get. I think we used iperf to test it and when we list all > the processes, the ksoftirq was using all the resource. Then I found this > page: https://blog.cloudflare.com/how-to-receive-a-million-packets/. I do > not remember the exact solution, but I guess when you configure your out > ethernet interface with the command below, it must work then. To my > understanding all the packets are processed in the same core in your > situation, because the port number is always the same. So, for example, if > you add another network with gtp-u tunnel on another port (different than > 3386) then again your packets will be processed on the other core, too. But > with the below command, the interface will be configured in a way that it > wont check the port to process on which core it should be processed, but it > will use the hash from the packet to distribute over the cores. > ethtool -n (your_out_eth_interface) rx-flow-hash udp4 > > Hope it will work you. > > Fırat > > Tony Clark <chiefy.padua(a)gmail.com>, 19 Haz 2019 Çar, 15:07 tarihinde > şunu yazdı: > >> Dear All, >> >> I've been using the GTP-U kernel module to communicate with a P-GW. >> >> Running Fedora 29, kernel 4.18.16-300.fc29.x86_64. >> >> At high traffic levels through the GTP-U tunnel I see the performance >> degrade as 100% CPU is consumed by a single ksoftirqd process. >> >> It is running on a multi-cpu machine and as far as I can tell the load is >> evenly spread across the cpus (ie either manually via smp_affinity, or even >> irqbalance, checking /proc/interrupts so forth.). >> >> Has anyone else experienced this? >> >> Is there any particular area you could recommend I investigate to find >> the root cause of this bottleneck, as i'm starting to scratch my head where >> to look next... >> >> Thanks in advance >> Tony >> >> ---- FYI >> >> modinfo gtp >> filename: >> /lib/modules/4.18.16-300.fc29.x86_64/kernel/drivers/net/gtp.ko.xz >> alias: net-pf-16-proto-16-family-gtp >> alias: rtnl-link-gtp >> description: Interface driver for GTP encapsulated traffic >> author: Harald Welte <hwelte(a)sysmocom.de> >> license: GPL >> depends: udp_tunnel >> retpoline: Y >> intree: Y >> name: gtp >> vermagic: 4.18.16-300.fc29.x86_64 SMP mod_unload >> >> modinfo udp_tunnel >> filename: >> /lib/modules/4.18.16-300.fc29.x86_64/kernel/net/ipv4/udp_tunnel.ko.xz >> license: GPL >> depends: >> retpoline: Y >> intree: Y >> name: udp_tunnel >> vermagic: 4.18.16-300.fc29.x86_64 SMP mod_unload >> >

2 1

EDGE MCS not stable
by ghiturbe＠tutanota.com 06 Nov '19

06 Nov '19

Dear all, Please do not consider the last message. First I want to introduce myslef, I'm Gael and few months ago I bought a Lime-Mini to use as BTS with EDGE. I have spent some time working in that and it is doing it but I have noticed that the UP LINK MCS is not stable, it changes very much with the mobile near the board. I have read that this issue was solved in the ticket 1833 but it is not working for me, I hope that this not about my board. Do you have any suggest for this behavior? To solve that issue, I force as I read in another post, setting the minimum an maximum values to zero till the MCS to be used. It works fine setting the pcu-cfg file like: ... mcs7 0 35 mcs8 35 35. It works but it does not look well and if I change by VTY to use mcs 9 it works too but is not posible to start the system with this configuration, so I have to start it using mcs7. With this cfg I can reach 130kbps using a mobile class 12 ( 4TS RX + TX), anyway I have 6 TS configured for PDCH. I guess this mobile should reach almost 200 kbps (4TS * 54Kbps). What is the maximum speed reached? Thanks in advance for your support. Best regards. -- Securely sent with Tutanota. Get your own encrypted, ad-free mailbox: https://tutanota.com

1 0

Correo confidencial de ghiturbe
by ghiturbe＠tutanota.com 06 Nov '19

06 Nov '19

1 0

[PATCH AUTOSEL 4.9 73/73] gtp: fix use-after-free in gtp_newlink()
by Sasha Levin 16 Jul '19

16 Jul '19

From: Taehee Yoo <ap420073(a)gmail.com> [ Upstream commit a2bed90704c68d3763bf24decb1b781a45395de8 ] Current gtp_newlink() could be called after unregister_pernet_subsys(). gtp_newlink() uses gtp_net but it can be destroyed by unregister_pernet_subsys(). So unregister_pernet_subsys() should be called after rtnl_link_unregister(). Test commands: #SHELL 1 while : do for i in {1..5} do ./gtp-link add gtp$i & done killall gtp-link done #SHELL 2 while : do modprobe -rv gtp done Splat looks like: [ 753.176631] BUG: KASAN: use-after-free in gtp_newlink+0x9b4/0xa5c [gtp] [ 753.177722] Read of size 8 at addr ffff8880d48f2458 by task gtp-link/7126 [ 753.179082] CPU: 0 PID: 7126 Comm: gtp-link Tainted: G W 5.2.0-rc6+ #50 [ 753.185801] Call Trace: [ 753.186264] dump_stack+0x7c/0xbb [ 753.186863] ? gtp_newlink+0x9b4/0xa5c [gtp] [ 753.187583] print_address_description+0xc7/0x240 [ 753.188382] ? gtp_newlink+0x9b4/0xa5c [gtp] [ 753.189097] ? gtp_newlink+0x9b4/0xa5c [gtp] [ 753.189846] __kasan_report+0x12a/0x16f [ 753.190542] ? gtp_newlink+0x9b4/0xa5c [gtp] [ 753.191298] kasan_report+0xe/0x20 [ 753.191893] gtp_newlink+0x9b4/0xa5c [gtp] [ 753.192580] ? __netlink_ns_capable+0xc3/0xf0 [ 753.193370] __rtnl_newlink+0xb9f/0x11b0 [ ... ] [ 753.241201] Allocated by task 7186: [ 753.241844] save_stack+0x19/0x80 [ 753.242399] __kasan_kmalloc.constprop.3+0xa0/0xd0 [ 753.243192] __kmalloc+0x13e/0x300 [ 753.243764] ops_init+0xd6/0x350 [ 753.244314] register_pernet_operations+0x249/0x6f0 [ ... ] [ 753.251770] Freed by task 7178: [ 753.252288] save_stack+0x19/0x80 [ 753.252833] __kasan_slab_free+0x111/0x150 [ 753.253962] kfree+0xc7/0x280 [ 753.254509] ops_free_list.part.11+0x1c4/0x2d0 [ 753.255241] unregister_pernet_operations+0x262/0x390 [ ... ] [ 753.285883] list_add corruption. next->prev should be prev (ffff8880d48f2458), but was ffff8880d497d878. (next. [ 753.287241] ------------[ cut here ]------------ [ 753.287794] kernel BUG at lib/list_debug.c:25! [ 753.288364] invalid opcode: 0000 [#1] SMP DEBUG_PAGEALLOC KASAN PTI [ 753.289099] CPU: 0 PID: 7126 Comm: gtp-link Tainted: G B W 5.2.0-rc6+ #50 [ 753.291036] RIP: 0010:__list_add_valid+0x74/0xd0 [ 753.291589] Code: 48 39 da 75 27 48 39 f5 74 36 48 39 dd 74 31 48 83 c4 08 b8 01 00 00 00 5b 5d c3 48 89 d9 48b [ 753.293779] RSP: 0018:ffff8880cae8f398 EFLAGS: 00010286 [ 753.294401] RAX: 0000000000000075 RBX: ffff8880d497d878 RCX: 0000000000000000 [ 753.296260] RDX: 0000000000000075 RSI: 0000000000000008 RDI: ffffed10195d1e69 [ 753.297070] RBP: ffff8880cd250ae0 R08: ffffed101b4bff21 R09: ffffed101b4bff21 [ 753.297899] R10: 0000000000000001 R11: ffffed101b4bff20 R12: ffff8880d497d878 [ 753.298703] R13: 0000000000000000 R14: ffff8880cd250ae0 R15: ffff8880d48f2458 [ 753.299564] FS: 00007f5f79805740(0000) GS:ffff8880da400000(0000) knlGS:0000000000000000 [ 753.300533] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 753.301231] CR2: 00007fe8c7ef4f10 CR3: 00000000b71a6006 CR4: 00000000000606f0 [ 753.302183] Call Trace: [ 753.302530] gtp_newlink+0x5f6/0xa5c [gtp] [ 753.303037] ? __netlink_ns_capable+0xc3/0xf0 [ 753.303576] __rtnl_newlink+0xb9f/0x11b0 [ 753.304092] ? rtnl_link_unregister+0x230/0x230 Fixes: 459aa660eb1d ("gtp: add initial driver for datapath of GPRS Tunneling Protocol (GTP-U)") Signed-off-by: Taehee Yoo <ap420073(a)gmail.com> Signed-off-by: David S. Miller <davem(a)davemloft.net> Signed-off-by: Sasha Levin <sashal(a)kernel.org> --- drivers/net/gtp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index 60df6e391ad2..7e1df403a37d 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -1358,9 +1358,9 @@ late_initcall(gtp_init); static void __exit gtp_fini(void) { - unregister_pernet_subsys(&gtp_net_ops); genl_unregister_family(&gtp_genl_family); rtnl_link_unregister(&gtp_link_ops); + unregister_pernet_subsys(&gtp_net_ops); pr_info("GTP module unloaded\n"); } -- 2.20.1

1 0

[PATCH AUTOSEL 4.9 72/73] gtp: fix Illegal context switch in RCU read-side critical section.
by Sasha Levin 16 Jul '19

16 Jul '19

From: Taehee Yoo <ap420073(a)gmail.com> [ Upstream commit 3f167e1921865b379a9becf03828e7202c7b4917 ] ipv4_pdp_add() is called in RCU read-side critical section. So GFP_KERNEL should not be used in the function. This patch make ipv4_pdp_add() to use GFP_ATOMIC instead of GFP_KERNEL. Test commands: gtp-link add gtp1 & gtp-tunnel add gtp1 v1 100 200 1.1.1.1 2.2.2.2 Splat looks like: [ 130.618881] ============================= [ 130.626382] WARNING: suspicious RCU usage [ 130.626994] 5.2.0-rc6+ #50 Not tainted [ 130.627622] ----------------------------- [ 130.628223] ./include/linux/rcupdate.h:266 Illegal context switch in RCU read-side critical section! [ 130.629684] [ 130.629684] other info that might help us debug this: [ 130.629684] [ 130.631022] [ 130.631022] rcu_scheduler_active = 2, debug_locks = 1 [ 130.632136] 4 locks held by gtp-tunnel/1025: [ 130.632925] #0: 000000002b93c8b7 (cb_lock){++++}, at: genl_rcv+0x15/0x40 [ 130.634159] #1: 00000000f17bc999 (genl_mutex){+.+.}, at: genl_rcv_msg+0xfb/0x130 [ 130.635487] #2: 00000000c644ed8e (rtnl_mutex){+.+.}, at: gtp_genl_new_pdp+0x18c/0x1150 [gtp] [ 130.636936] #3: 0000000007a1cde7 (rcu_read_lock){....}, at: gtp_genl_new_pdp+0x187/0x1150 [gtp] [ 130.638348] [ 130.638348] stack backtrace: [ 130.639062] CPU: 1 PID: 1025 Comm: gtp-tunnel Not tainted 5.2.0-rc6+ #50 [ 130.641318] Call Trace: [ 130.641707] dump_stack+0x7c/0xbb [ 130.642252] ___might_sleep+0x2c0/0x3b0 [ 130.642862] kmem_cache_alloc_trace+0x1cd/0x2b0 [ 130.643591] gtp_genl_new_pdp+0x6c5/0x1150 [gtp] [ 130.644371] genl_family_rcv_msg+0x63a/0x1030 [ 130.645074] ? mutex_lock_io_nested+0x1090/0x1090 [ 130.645845] ? genl_unregister_family+0x630/0x630 [ 130.646592] ? debug_show_all_locks+0x2d0/0x2d0 [ 130.647293] ? check_flags.part.40+0x440/0x440 [ 130.648099] genl_rcv_msg+0xa3/0x130 [ ... ] Fixes: 459aa660eb1d ("gtp: add initial driver for datapath of GPRS Tunneling Protocol (GTP-U)") Signed-off-by: Taehee Yoo <ap420073(a)gmail.com> Signed-off-by: David S. Miller <davem(a)davemloft.net> Signed-off-by: Sasha Levin <sashal(a)kernel.org> --- drivers/net/gtp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index cb206e5526c4..60df6e391ad2 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -952,7 +952,7 @@ static int ipv4_pdp_add(struct net_device *dev, struct genl_info *info) } - pctx = kmalloc(sizeof(struct pdp_ctx), GFP_KERNEL); + pctx = kmalloc(sizeof(*pctx), GFP_ATOMIC); if (pctx == NULL) return -ENOMEM; -- 2.20.1

1 0

[PATCH AUTOSEL 4.14 104/105] gtp: fix use-after-free in gtp_newlink()
by Sasha Levin 16 Jul '19

16 Jul '19

From: Taehee Yoo <ap420073(a)gmail.com> [ Upstream commit a2bed90704c68d3763bf24decb1b781a45395de8 ] Current gtp_newlink() could be called after unregister_pernet_subsys(). gtp_newlink() uses gtp_net but it can be destroyed by unregister_pernet_subsys(). So unregister_pernet_subsys() should be called after rtnl_link_unregister(). Test commands: #SHELL 1 while : do for i in {1..5} do ./gtp-link add gtp$i & done killall gtp-link done #SHELL 2 while : do modprobe -rv gtp done Splat looks like: [ 753.176631] BUG: KASAN: use-after-free in gtp_newlink+0x9b4/0xa5c [gtp] [ 753.177722] Read of size 8 at addr ffff8880d48f2458 by task gtp-link/7126 [ 753.179082] CPU: 0 PID: 7126 Comm: gtp-link Tainted: G W 5.2.0-rc6+ #50 [ 753.185801] Call Trace: [ 753.186264] dump_stack+0x7c/0xbb [ 753.186863] ? gtp_newlink+0x9b4/0xa5c [gtp] [ 753.187583] print_address_description+0xc7/0x240 [ 753.188382] ? gtp_newlink+0x9b4/0xa5c [gtp] [ 753.189097] ? gtp_newlink+0x9b4/0xa5c [gtp] [ 753.189846] __kasan_report+0x12a/0x16f [ 753.190542] ? gtp_newlink+0x9b4/0xa5c [gtp] [ 753.191298] kasan_report+0xe/0x20 [ 753.191893] gtp_newlink+0x9b4/0xa5c [gtp] [ 753.192580] ? __netlink_ns_capable+0xc3/0xf0 [ 753.193370] __rtnl_newlink+0xb9f/0x11b0 [ ... ] [ 753.241201] Allocated by task 7186: [ 753.241844] save_stack+0x19/0x80 [ 753.242399] __kasan_kmalloc.constprop.3+0xa0/0xd0 [ 753.243192] __kmalloc+0x13e/0x300 [ 753.243764] ops_init+0xd6/0x350 [ 753.244314] register_pernet_operations+0x249/0x6f0 [ ... ] [ 753.251770] Freed by task 7178: [ 753.252288] save_stack+0x19/0x80 [ 753.252833] __kasan_slab_free+0x111/0x150 [ 753.253962] kfree+0xc7/0x280 [ 753.254509] ops_free_list.part.11+0x1c4/0x2d0 [ 753.255241] unregister_pernet_operations+0x262/0x390 [ ... ] [ 753.285883] list_add corruption. next->prev should be prev (ffff8880d48f2458), but was ffff8880d497d878. (next. [ 753.287241] ------------[ cut here ]------------ [ 753.287794] kernel BUG at lib/list_debug.c:25! [ 753.288364] invalid opcode: 0000 [#1] SMP DEBUG_PAGEALLOC KASAN PTI [ 753.289099] CPU: 0 PID: 7126 Comm: gtp-link Tainted: G B W 5.2.0-rc6+ #50 [ 753.291036] RIP: 0010:__list_add_valid+0x74/0xd0 [ 753.291589] Code: 48 39 da 75 27 48 39 f5 74 36 48 39 dd 74 31 48 83 c4 08 b8 01 00 00 00 5b 5d c3 48 89 d9 48b [ 753.293779] RSP: 0018:ffff8880cae8f398 EFLAGS: 00010286 [ 753.294401] RAX: 0000000000000075 RBX: ffff8880d497d878 RCX: 0000000000000000 [ 753.296260] RDX: 0000000000000075 RSI: 0000000000000008 RDI: ffffed10195d1e69 [ 753.297070] RBP: ffff8880cd250ae0 R08: ffffed101b4bff21 R09: ffffed101b4bff21 [ 753.297899] R10: 0000000000000001 R11: ffffed101b4bff20 R12: ffff8880d497d878 [ 753.298703] R13: 0000000000000000 R14: ffff8880cd250ae0 R15: ffff8880d48f2458 [ 753.299564] FS: 00007f5f79805740(0000) GS:ffff8880da400000(0000) knlGS:0000000000000000 [ 753.300533] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 753.301231] CR2: 00007fe8c7ef4f10 CR3: 00000000b71a6006 CR4: 00000000000606f0 [ 753.302183] Call Trace: [ 753.302530] gtp_newlink+0x5f6/0xa5c [gtp] [ 753.303037] ? __netlink_ns_capable+0xc3/0xf0 [ 753.303576] __rtnl_newlink+0xb9f/0x11b0 [ 753.304092] ? rtnl_link_unregister+0x230/0x230 Fixes: 459aa660eb1d ("gtp: add initial driver for datapath of GPRS Tunneling Protocol (GTP-U)") Signed-off-by: Taehee Yoo <ap420073(a)gmail.com> Signed-off-by: David S. Miller <davem(a)davemloft.net> Signed-off-by: Sasha Levin <sashal(a)kernel.org> --- drivers/net/gtp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index 53fd66534e3a..5de4053774b8 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -1383,9 +1383,9 @@ late_initcall(gtp_init); static void __exit gtp_fini(void) { - unregister_pernet_subsys(&gtp_net_ops); genl_unregister_family(&gtp_genl_family); rtnl_link_unregister(&gtp_link_ops); + unregister_pernet_subsys(&gtp_net_ops); pr_info("GTP module unloaded\n"); } -- 2.20.1

1 0

← Newer
1
...
14
15
16
17
18
19
20
...
49
Older →

Jump to page:

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

osmocom-net-gprs