diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
-index f184fb5..0e08522 100644
+index 95c32f2..93fada5 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
-@@ -234,6 +234,125 @@ config RIONET_RX_SIZE
+@@ -260,6 +260,125 @@ config RIONET_RX_SIZE
depends on RIONET
default "128"
tristate "Universal TUN/TAP device driver support"
depends on INET
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
-index 900b0c5..e093402 100644
+index 7336cbd..d6d7ad4 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
-@@ -10,6 +10,7 @@ obj-$(CONFIG_IPVLAN) += ipvlan/
- obj-$(CONFIG_DUMMY) += dummy.o
+@@ -11,6 +11,7 @@ obj-$(CONFIG_DUMMY) += dummy.o
obj-$(CONFIG_EQUALIZER) += eql.o
obj-$(CONFIG_IFB) += ifb.o
+ obj-$(CONFIG_MACSEC) += macsec.o
+obj-$(CONFIG_IMQ) += imq.o
obj-$(CONFIG_MACVLAN) += macvlan.o
obj-$(CONFIG_MACVTAP) += macvtap.o
obj-$(CONFIG_MII) += mii.o
diff --git a/drivers/net/imq.c b/drivers/net/imq.c
new file mode 100644
-index 0000000..f80258f
+index 0000000..bc3b997
--- /dev/null
+++ b/drivers/net/imq.c
-@@ -0,0 +1,903 @@
+@@ -0,0 +1,907 @@
+/*
+ * Pseudo-driver for the intermediate queue device.
+ *
+ struct nf_queue_entry *entry = skb->nf_queue_entry;
+
+ skb->nf_queue_entry = NULL;
-+ dev->trans_start = jiffies;
++ netif_trans_update(dev);
+
+ dev->stats.tx_bytes += skb->len;
+ dev->stats.tx_packets++;
+{
+ struct sk_buff *skb_orig, *skb, *skb_shared, *skb_popd;
+ struct Qdisc *q;
++ struct sk_buff *to_free = NULL;
+ struct netdev_queue *txq;
+ spinlock_t *root_lock;
+ int users;
+
+ /* backup skb->cb, as qdisc layer will overwrite it */
+ skb_save_cb(skb_shared);
-+ qdisc_enqueue_root(skb_shared, q); /* might kfree_skb */
++ qdisc_enqueue_root(skb_shared, q, &to_free); /* might kfree_skb */
+ if (likely(atomic_read(&skb_shared->users) == users + 1)) {
+ bool validate;
+
+ /* Note that we validate skb (GSO, checksum, ...) outside of locks */
+ if (validate)
+ skb_popd = validate_xmit_skb_list(skb_popd, dev);
-+
++
+ if (skb_popd) {
+ int dummy_ret;
+ int cpu = smp_processor_id(); /* ok because BHs are off */
+
+ txq = skb_get_tx_queue(dev, skb_popd);
-+ /*
++ /*
+ IMQ device will not be frozen or stoped, and it always be successful.
+ So we need not check its status and return value to accelerate.
+ */
+ }
+ retval = -1;
+out:
++ if (unlikely(to_free)) {
++ kfree_skb_list(to_free);
++ }
+ return retval;
+}
+static unsigned int imq_nf_hook(void *priv,
+MODULE_PARM_DESC(numqueues, "number of queues per IMQ device");
+MODULE_PARM_DESC(imq_dev_accurate_stats, "Notify if need the accurate imq device stats");
+
-+MODULE_AUTHOR("http://https://github.com/imq/linuximq");
++MODULE_AUTHOR("https://github.com/imq/linuximq");
+MODULE_DESCRIPTION("Pseudo-driver for the intermediate queue device. See https://github.com/imq/linuximq/wiki for more information.");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS_RTNL_LINK("imq");
+#endif /* _IMQ_H */
+
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
-index 3143c84..e213b31 100644
+index e16a2a9..4a1090a 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
-@@ -3341,6 +3341,19 @@ static inline void netif_tx_unlock_bh(struct net_device *dev)
+@@ -3669,6 +3669,19 @@ static inline void netif_tx_unlock_bh(struct net_device *dev)
} \
}
+#endif /* _IP6T_IMQ_H */
+
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
-index 4355129..47914c0 100644
+index 32810f2..4ce1d0a 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
-@@ -38,6 +38,10 @@
- #include <linux/splice.h>
+@@ -39,6 +39,10 @@
#include <linux/in6.h>
+ #include <linux/if_packet.h>
#include <net/flow.h>
+#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
+#include <linux/imq.h>
+#endif
+
- /* A. Checksumming of received packets by device.
- *
-@@ -566,6 +570,9 @@ struct sk_buff {
+ /* The interface for checksum offload between the stack and networking drivers
+ * is as follows...
+@@ -654,6 +658,9 @@ struct sk_buff {
* first. This is owned by whoever has the skb queued ATM.
*/
char cb[48] __aligned(8);
unsigned long _skb_refdst;
void (*destructor)(struct sk_buff *skb);
-@@ -575,6 +582,9 @@ struct sk_buff {
+@@ -663,6 +670,9 @@ struct sk_buff {
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
struct nf_conntrack *nfct;
#endif
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
struct nf_bridge_info *nf_bridge;
#endif
-@@ -642,6 +652,9 @@ struct sk_buff {
- __u8 inner_protocol_type:1;
- __u8 remcsum_offload:1;
- /* 3 or 5 bit hole */
-+ #if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
-+ __u8 imq_flags:IMQ_F_BITS;
-+ #endif
+@@ -743,6 +753,9 @@ struct sk_buff {
+ __u8 offload_fwd_mark:1;
+ #endif
+ /* 2, 4 or 5 bit hole */
++#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
++ __u8 imq_flags:IMQ_F_BITS;
++#endif
#ifdef CONFIG_NET_SCHED
__u16 tc_index; /* traffic control index */
-@@ -798,6 +811,12 @@ void kfree_skb_list(struct sk_buff *segs);
+@@ -903,6 +916,12 @@ void kfree_skb_list(struct sk_buff *segs);
void skb_tx_error(struct sk_buff *skb);
void consume_skb(struct sk_buff *skb);
void __kfree_skb(struct sk_buff *skb);
extern struct kmem_cache *skbuff_head_cache;
void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
-@@ -3344,6 +3363,10 @@ static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
+@@ -3594,6 +3613,10 @@ static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
if (copy)
dst->nfctinfo = src->nfctinfo;
#endif
dst->nf_bridge = src->nf_bridge;
nf_bridge_get(src->nf_bridge);
diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h
-index 9c5638a..b173aa7 100644
+index 2280cfe..ec8fa51 100644
--- a/include/net/netfilter/nf_queue.h
+++ b/include/net/netfilter/nf_queue.h
-@@ -31,6 +31,12 @@ struct nf_queue_handler {
- void nf_register_queue_handler(const struct nf_queue_handler *qh);
- void nf_unregister_queue_handler(void);
+@@ -30,6 +30,12 @@ struct nf_queue_handler {
+ void nf_register_queue_handler(struct net *net, const struct nf_queue_handler *qh);
+ void nf_unregister_queue_handler(struct net *net);
void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict);
+void nf_queue_entry_release_refs(struct nf_queue_entry *entry);
+
void nf_queue_entry_get_refs(struct nf_queue_entry *entry);
void nf_queue_entry_release_refs(struct nf_queue_entry *entry);
diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h
-index 401038d..4668849 100644
+index cd334c9..6757228 100644
--- a/include/net/pkt_sched.h
+++ b/include/net/pkt_sched.h
-@@ -104,6 +104,8 @@ int sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
+@@ -105,6 +105,8 @@ int sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
void __qdisc_run(struct Qdisc *q);
{
if (qdisc_run_begin(q))
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
-index b2a8e63..d9feaa3 100644
+index e6aa0a2..08b37dc 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
-@@ -506,6 +506,12 @@ static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
- return sch->enqueue(skb, sch);
+@@ -518,6 +518,13 @@ static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+ return sch->enqueue(skb, sch, to_free);
}
-+static inline int qdisc_enqueue_root(struct sk_buff *skb, struct Qdisc *sch)
++static inline int qdisc_enqueue_root(struct sk_buff *skb, struct Qdisc *sch,
++ struct sk_buff **to_free)
+{
+ qdisc_skb_cb(skb)->pkt_len = skb->len;
-+ return qdisc_enqueue(skb, sch) & NET_XMIT_MASK;
++ return qdisc_enqueue(skb, sch, to_free) & NET_XMIT_MASK;
+}
+
static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
/* we overload the higher bits for encoding auxiliary data such as the queue
* number or errno values. Not nice, but better than additional function
diff --git a/net/core/dev.c b/net/core/dev.c
-index ae00b89..1cdcd02 100644
+index 6666b28..3e12add 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
-@@ -137,6 +137,9 @@
- #include <linux/errqueue.h>
- #include <linux/hrtimer.h>
+@@ -141,6 +141,9 @@
#include <linux/netfilter_ingress.h>
+ #include <linux/sctp.h>
+ #include <linux/crash_dump.h>
+#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
+#include <linux/imq.h>
+#endif
#include "net-sysfs.h"
-@@ -2705,7 +2708,12 @@ static int xmit_one(struct sk_buff *skb, struct net_device *dev,
+@@ -2906,7 +2909,12 @@ static int xmit_one(struct sk_buff *skb, struct net_device *dev,
unsigned int len;
int rc;
dev_queue_xmit_nit(skb, dev);
len = skb->len;
-@@ -2743,6 +2751,7 @@ out:
- *ret = rc;
+@@ -2945,6 +2953,8 @@ struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *de
return skb;
}
-+EXPORT_SYMBOL(dev_hard_start_xmit);
++EXPORT_SYMBOL_GPL(dev_hard_start_xmit);
++
static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
netdev_features_t features)
-@@ -2831,6 +2840,7 @@ struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *d
- }
- return head;
- }
-+EXPORT_SYMBOL(validate_xmit_skb_list);
-
- static void qdisc_pkt_len_init(struct sk_buff *skb)
{
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
-index b2df375..bc3c51e 100644
+index 1e3e008..379236e 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
-@@ -79,6 +79,87 @@
+@@ -82,6 +82,87 @@ struct kmem_cache *skbuff_head_cache __read_mostly;
static struct kmem_cache *skbuff_fclone_cache __read_mostly;
int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
EXPORT_SYMBOL(sysctl_max_skb_frags);
/**
* skb_panic - private function for out-of-line support
-@@ -643,6 +724,28 @@ static void skb_release_head_state(struct sk_buff *skb)
+@@ -654,6 +735,28 @@ static void skb_release_head_state(struct sk_buff *skb)
WARN_ON(in_irq());
skb->destructor(skb);
}
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
nf_conntrack_put(skb->nfct);
#endif
-@@ -765,6 +868,10 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
+@@ -843,6 +946,10 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
new->sp = secpath_get(old->sp);
#endif
__nf_copy(new, old, false);
/* Note : this field could be in headers_start/headers_end section
* It is not yet because we do not want to have a 16 bit hole
-@@ -3325,6 +3432,13 @@ void __init skb_init(void)
+@@ -3463,6 +3570,13 @@ void __init skb_init(void)
0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC,
NULL);
/**
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
-index e6a7bd15..c81496e 100644
+index 59eb4ed..8020b07 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
-@@ -65,9 +65,6 @@ static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *
+@@ -66,9 +66,6 @@ static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *
struct in6_addr *nexthop;
int ret;
if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) {
struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
-@@ -142,6 +139,13 @@ int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
+@@ -150,6 +147,13 @@ int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
return 0;
}
net, sk, skb, NULL, dev,
ip6_finish_output,
diff --git a/net/netfilter/Kconfig b/net/netfilter/Kconfig
-index 4692782..dce47db 100644
+index e8d56d9..1ed3468 100644
--- a/net/netfilter/Kconfig
+++ b/net/netfilter/Kconfig
-@@ -785,6 +785,18 @@ config NETFILTER_XT_TARGET_LOG
+@@ -823,6 +823,18 @@ config NETFILTER_XT_TARGET_LOG
To compile it as a module, choose M here. If unsure, say N.
tristate '"MARK" target support'
depends on NETFILTER_ADVANCED
diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile
-index 7638c36..614ad8a 100644
+index c23c3c8..99911ef 100644
--- a/net/netfilter/Makefile
+++ b/net/netfilter/Makefile
-@@ -108,6 +108,7 @@ obj-$(CONFIG_NETFILTER_XT_TARGET_CT) += xt_CT.o
+@@ -119,6 +119,7 @@ obj-$(CONFIG_NETFILTER_XT_TARGET_CT) += xt_CT.o
obj-$(CONFIG_NETFILTER_XT_TARGET_DSCP) += xt_DSCP.o
obj-$(CONFIG_NETFILTER_XT_TARGET_HL) += xt_HL.o
obj-$(CONFIG_NETFILTER_XT_TARGET_HMARK) += xt_HMARK.o
obj-$(CONFIG_NETFILTER_XT_TARGET_LOG) += xt_LOG.o
obj-$(CONFIG_NETFILTER_XT_TARGET_NETMAP) += xt_NETMAP.o
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
-index f39276d..9877a27 100644
+index 004af03..768a08b 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
-@@ -311,9 +311,11 @@ next_hook:
+@@ -360,8 +360,11 @@ int nf_hook_slow(struct sk_buff *skb, struct nf_hook_state *state)
ret = NF_DROP_GETERR(verdict);
if (ret == 0)
ret = -EPERM;
- } else if ((verdict & NF_VERDICT_MASK) == NF_QUEUE) {
+ } else if ((verdict & NF_VERDICT_MASK) == NF_QUEUE ||
-+ (verdict & NF_VERDICT_MASK) == NF_IMQ_QUEUE) {
- int err = nf_queue(skb, elem, state,
-- verdict >> NF_VERDICT_QBITS);
-+ verdict >> NF_VERDICT_QBITS,
-+ verdict & NF_VERDICT_MASK);
- if (err < 0) {
- if (err == -ESRCH &&
- (verdict & NF_VERDICT_FLAG_QUEUE_BYPASS))
-diff --git a/net/netfilter/nf_internals.h b/net/netfilter/nf_internals.h
-index 0655225..25d4141 100644
---- a/net/netfilter/nf_internals.h
-+++ b/net/netfilter/nf_internals.h
-@@ -18,7 +18,7 @@ unsigned int nf_iterate(struct list_head *head, struct sk_buff *skb,
-
- /* nf_queue.c */
- int nf_queue(struct sk_buff *skb, struct nf_hook_ops *elem,
-- struct nf_hook_state *state, unsigned int queuenum);
-+ struct nf_hook_state *state, unsigned int queuenum, unsigned int queuetype);
- void nf_queue_nf_hook_drop(struct net *net, struct nf_hook_ops *ops);
- int __init netfilter_queue_init(void);
-
++ (verdict & NF_VERDICT_MASK) == NF_IMQ_QUEUE) {
+ ret = nf_queue(skb, state, &entry, verdict);
++ if (ret == -ECANCELED)
++ goto next_hook;
+ if (ret == 1 && entry)
+ goto next_hook;
+ }
diff --git a/net/netfilter/nf_queue.c b/net/netfilter/nf_queue.c
-index 5baa8e2..9740e8c 100644
+index 8f08d75..8d362c0 100644
--- a/net/netfilter/nf_queue.c
+++ b/net/netfilter/nf_queue.c
-@@ -28,6 +28,23 @@
+@@ -27,6 +27,23 @@
+ * receives, no matter what.
*/
- static const struct nf_queue_handler __rcu *queue_handler __read_mostly;
+#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
+static const struct nf_queue_handler __rcu *queue_imq_handler __read_mostly;
+
/* return EBUSY when somebody else is registered, return EEXIST if the
* same handler is registered, return 0 in case of success. */
- void nf_register_queue_handler(const struct nf_queue_handler *qh)
-@@ -116,7 +133,8 @@ void nf_queue_nf_hook_drop(struct net *net, struct nf_hook_ops *ops)
- int nf_queue(struct sk_buff *skb,
- struct nf_hook_ops *elem,
- struct nf_hook_state *state,
-- unsigned int queuenum)
-+ unsigned int queuenum,
-+ unsigned int queuetype)
+ void nf_register_queue_handler(struct net *net, const struct nf_queue_handler *qh)
+@@ -108,16 +125,28 @@ void nf_queue_nf_hook_drop(struct net *net, const struct nf_hook_entry *entry)
+ }
+
+ static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state,
+- unsigned int queuenum)
++ unsigned int verdict)
{
int status = -ENOENT;
struct nf_queue_entry *entry = NULL;
-@@ -124,7 +142,17 @@ int nf_queue(struct sk_buff *skb,
+ const struct nf_afinfo *afinfo;
const struct nf_queue_handler *qh;
+ struct net *net = state->net;
++ unsigned int queuetype = verdict & NF_VERDICT_MASK;
++ unsigned int queuenum = verdict >> NF_VERDICT_QBITS;
/* QUEUE == DROP if no one is waiting, to be safe. */
-- qh = rcu_dereference(queue_handler);
+- qh = rcu_dereference(net->nf.queue_handler);
+ if (queuetype == NF_IMQ_QUEUE) {
+#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
+ qh = rcu_dereference(queue_imq_handler);
+ goto err_unlock;
+#endif
+ } else {
-+ qh = rcu_dereference(queue_handler);
++ qh = rcu_dereference(net->nf.queue_handler);
+ }
+
if (!qh) {
status = -ESRCH;
goto err;
-@@ -199,8 +227,10 @@ void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict)
+@@ -218,6 +247,7 @@ void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict)
local_bh_enable();
break;
case NF_QUEUE:
+ case NF_IMQ_QUEUE:
- err = nf_queue(skb, elem, &entry->state,
-- verdict >> NF_VERDICT_QBITS);
-+ verdict >> NF_VERDICT_QBITS,
-+ verdict & NF_VERDICT_MASK);
- if (err < 0) {
- if (err == -ESRCH &&
- (verdict & NF_VERDICT_FLAG_QUEUE_BYPASS))
+ err = nf_queue(skb, &entry->state, &hook_entry, verdict);
+ if (err == 1) {
+ if (hook_entry)
diff --git a/net/netfilter/xt_IMQ.c b/net/netfilter/xt_IMQ.c
new file mode 100644
-index 0000000..86d7b84
+index 0000000..f9c5817
--- /dev/null
+++ b/net/netfilter/xt_IMQ.c
@@ -0,0 +1,72 @@
+module_init(imq_init);
+module_exit(imq_fini);
+
-+MODULE_AUTHOR("http://https://github.com/imq/linuximq");
++MODULE_AUTHOR("https://github.com/imq/linuximq");
+MODULE_DESCRIPTION("Pseudo-driver for the intermediate queue device. See https://github.com/imq/linuximq/wiki for more information.");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("ipt_IMQ");
+MODULE_ALIAS("ip6t_IMQ");
+
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
-index e82a1ad..b55331d 100644
+index 6cfb6e9..4c675e9 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
-@@ -108,6 +108,14 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
+@@ -154,6 +154,14 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
return skb;
}
+}
+EXPORT_SYMBOL(qdisc_dequeue_skb);
+
- static inline int handle_dev_cpu_collision(struct sk_buff *skb,
- struct netdev_queue *dev_queue,
- struct Qdisc *q)
+ /*
+ * Transmit possibly several skbs, and handle the return status as
+ * required. Owning running seqcount bit guarantees that
projects / packages / kernel.git / blobdiff