« back to all changes in this revision
Viewing changes to lib/dpif-netdev.c
- browse files at revision 70
- compare with another revision
- download diff
- download tarball
- view history from revision 70
- Committer: Package Import Robot
- Author(s): James Page
- Date: 2015-08-10 11:35:15 UTC
- mfrom: (1.1.30)
- Revision ID: package-import@ubuntu.com-20150810113515-575vj06oq29emxsn
Tags: 2.4.0~git20150810.97bab95-0ubuntu1
* New upstream snapshot from 2.4 branch:
- d/*: Align any relevant packaging changes with upstream.
* d/*: wrap-and-sort.
* d/openvswitch-{common,vswitch}.install: Correct install location for
bash completion files.
* d/tests/openflow.py: Explicitly use ovs-testcontroller as provided
by 2.4.0 release.
- d/*: Align any relevant packaging changes with upstream.
* d/*: wrap-and-sort.
* d/openvswitch-{common,vswitch}.install: Correct install location for
bash completion files.
* d/tests/openflow.py: Explicitly use ovs-testcontroller as provided
by 2.4.0 release.
- files added:
- .gitignore
- .travis
- Documentation
- datapath-windows
- datapath-windows/Package
- datapath-windows/include
- datapath-windows/misc
- datapath-windows/ovsext
- datapath-windows/ovsext/Netlink
- datapath/linux/compat/build-aux
- include/windows/linux
- include/windows/netpacket
- lib/lldp
- python/build
- tests/dpdk
- windows
- windows/ovs-windows-installer
- windows/ovs-windows-installer/Actions
- windows/ovs-windows-installer/Binaries
- windows/ovs-windows-installer/Dialogs
- windows/ovs-windows-installer/Driver
- windows/ovs-windows-installer/Driver/Win8
- windows/ovs-windows-installer/Driver/Win8.1
- windows/ovs-windows-installer/Redist
- windows/ovs-windows-installer/Services
- windows/ovs-windows-installer/Symbols
- windows/ovs-windows-installer/images
- files removed:
- .pc/cfm-avoid-long-delay.patch
- .pc/cfm-avoid-long-delay.patch/lib
- .pc/disable-poll-loop-logging-for-long-output.patch
- .pc/disable-poll-loop-logging-for-long-output.patch/tests
- .pc/ovs-appctl-log-options.patch
- .pc/ovs-appctl-log-options.patch/utilities
- .pc/xcp-interface-reconfigure.patch
- .pc/xcp-interface-reconfigure.patch/xenserver
- datapath/linux/compat/include/asm
- debian/patches
- include/linux
- files modified:
- .pc/applied-patches
added
removed
lib/dpif-netdev.c
15
15
*/
16
16
17
17
#include <config.h>
18
#include "dpif.h"
18
#include "dpif-netdev.h"
19
19
20
20
#include <ctype.h>
21
21
#include <errno.h>
31
31
#include <sys/stat.h>
32
32
#include <unistd.h>
33
33
34
#include "classifier.h"
34
#include "cmap.h"
35
35
#include "csum.h"
36
#include "dp-packet.h"
36
37
#include "dpif.h"
37
38
#include "dpif-provider.h"
38
39
#include "dummy.h"
39
40
#include "dynamic-string.h"
41
#include "fat-rwlock.h"
40
42
#include "flow.h"
41
#include "hmap.h"
43
#include "cmap.h"
42
44
#include "latch.h"
43
45
#include "list.h"
46
#include "match.h"
44
47
#include "meta-flow.h"
45
48
#include "netdev.h"
46
49
#include "netdev-dpdk.h"
50
53
#include "odp-util.h"
51
54
#include "ofp-print.h"
52
55
#include "ofpbuf.h"
56
#include "ovs-numa.h"
53
57
#include "ovs-rcu.h"
54
58
#include "packets.h"
55
59
#include "poll-loop.h"
60
#include "pvector.h"
56
61
#include "random.h"
57
62
#include "seq.h"
58
63
#include "shash.h"
59
64
#include "sset.h"
60
65
#include "timeval.h"
66
#include "tnl-arp-cache.h"
61
67
#include "unixctl.h"
62
68
#include "util.h"
63
#include "vlog.h"
69
#include "openvswitch/vlog.h"
64
70
65
71
VLOG_DEFINE_THIS_MODULE(dpif_netdev);
66
72
67
/* By default, choose a priority in the middle. */
68
#define NETDEV_RULE_PRIORITY 0x8000
69
70
#define NR_THREADS 1
73
#define FLOW_DUMP_MAX_BATCH 50
71
74
/* Use per thread recirc_depth to prevent recirculation loop. */
72
75
#define MAX_RECIRC_DEPTH 5
73
76
DEFINE_STATIC_PER_THREAD_DATA(uint32_t, recirc_depth, 0)
75
78
/* Configuration parameters. */
76
79
enum { MAX_FLOWS = 65536 }; /* Maximum number of flows in flow table. */
77
80
78
/* Queues. */
79
enum { MAX_QUEUE_LEN = 128 }; /* Maximum number of packets per queue. */
80
enum { QUEUE_MASK = MAX_QUEUE_LEN - 1 };
81
BUILD_ASSERT_DECL(IS_POW2(MAX_QUEUE_LEN));
82
83
81
/* Protects against changes to 'dp_netdevs'. */
84
82
static struct ovs_mutex dp_netdev_mutex = OVS_MUTEX_INITIALIZER;
85
83
87
85
static struct shash dp_netdevs OVS_GUARDED_BY(dp_netdev_mutex)
88
86
= SHASH_INITIALIZER(&dp_netdevs);
89
87
90
struct dp_netdev_upcall {
91
struct dpif_upcall upcall; /* Queued upcall information. */
92
struct ofpbuf buf; /* ofpbuf instance for upcall.packet. */
88
static struct vlog_rate_limit upcall_rl = VLOG_RATE_LIMIT_INIT(600, 600);
89
90
/* Stores a miniflow with inline values */
91
92
struct netdev_flow_key {
93
uint32_t hash; /* Hash function differs for different users. */
94
uint32_t len; /* Length of the following miniflow (incl. map). */
95
struct miniflow mf;
96
uint64_t buf[FLOW_MAX_PACKET_U64S - MINI_N_INLINE];
93
97
};
94
98
95
/* A queue passing packets from a struct dp_netdev to its clients (handlers).
99
/* Exact match cache for frequently used flows
100
*
101
* The cache uses a 32-bit hash of the packet (which can be the RSS hash) to
102
* search its entries for a miniflow that matches exactly the miniflow of the
103
* packet. It stores the 'dpcls_rule' (rule) that matches the miniflow.
104
*
105
* A cache entry holds a reference to its 'dp_netdev_flow'.
106
*
107
* A miniflow with a given hash can be in one of EM_FLOW_HASH_SEGS different
108
* entries. The 32-bit hash is split into EM_FLOW_HASH_SEGS values (each of
109
* them is EM_FLOW_HASH_SHIFT bits wide and the remainder is thrown away). Each
110
* value is the index of a cache entry where the miniflow could be.
96
111
*
97
112
*
98
113
* Thread-safety
99
114
* =============
100
115
*
101
* Any access at all requires the owning 'dp_netdev''s queue_rwlock and
102
* its own mutex. */
103
struct dp_netdev_queue {
104
struct ovs_mutex mutex;
105
struct seq *seq; /* Incremented whenever a packet is queued. */
106
struct dp_netdev_upcall upcalls[MAX_QUEUE_LEN] OVS_GUARDED;
107
unsigned int head OVS_GUARDED;
108
unsigned int tail OVS_GUARDED;
109
};
110
116
* Each pmd_thread has its own private exact match cache.
117
* If dp_netdev_input is not called from a pmd thread, a mutex is used.
118
*/
119
120
#define EM_FLOW_HASH_SHIFT 13
121
#define EM_FLOW_HASH_ENTRIES (1u << EM_FLOW_HASH_SHIFT)
122
#define EM_FLOW_HASH_MASK (EM_FLOW_HASH_ENTRIES - 1)
123
#define EM_FLOW_HASH_SEGS 2
124
125
struct emc_entry {
126
struct dp_netdev_flow *flow;
127
struct netdev_flow_key key; /* key.hash used for emc hash value. */
128
};
129
130
struct emc_cache {
131
struct emc_entry entries[EM_FLOW_HASH_ENTRIES];
132
int sweep_idx; /* For emc_cache_slow_sweep(). */
133
};
134
135
/* Iterate in the exact match cache through every entry that might contain a
136
* miniflow with hash 'HASH'. */
137
#define EMC_FOR_EACH_POS_WITH_HASH(EMC, CURRENT_ENTRY, HASH) \
138
for (uint32_t i__ = 0, srch_hash__ = (HASH); \
139
(CURRENT_ENTRY) = &(EMC)->entries[srch_hash__ & EM_FLOW_HASH_MASK], \
140
i__ < EM_FLOW_HASH_SEGS; \
141
i__++, srch_hash__ >>= EM_FLOW_HASH_SHIFT)
142
143
/* Simple non-wildcarding single-priority classifier. */
144
145
struct dpcls {
146
struct cmap subtables_map;
147
struct pvector subtables;
148
};
149
150
/* A rule to be inserted to the classifier. */
151
struct dpcls_rule {
152
struct cmap_node cmap_node; /* Within struct dpcls_subtable 'rules'. */
153
struct netdev_flow_key *mask; /* Subtable's mask. */
154
struct netdev_flow_key flow; /* Matching key. */
155
/* 'flow' must be the last field, additional space is allocated here. */
156
};
157
158
static void dpcls_init(struct dpcls *);
159
static void dpcls_destroy(struct dpcls *);
160
static void dpcls_insert(struct dpcls *, struct dpcls_rule *,
161
const struct netdev_flow_key *mask);
162
static void dpcls_remove(struct dpcls *, struct dpcls_rule *);
163
static bool dpcls_lookup(const struct dpcls *cls,
164
const struct netdev_flow_key keys[],
165
struct dpcls_rule **rules, size_t cnt);
166
111
167
/* Datapath based on the network device interface from netdev.h.
112
168
*
113
169
*
120
176
* Acquisition order is, from outermost to innermost:
121
177
*
122
178
* dp_netdev_mutex (global)
123
* port_rwlock
124
* flow_mutex
125
* cls.rwlock
126
* queue_rwlock
179
* port_mutex
127
180
*/
128
181
struct dp_netdev {
129
182
const struct dpif_class *const class;
130
183
const char *const name;
184
struct dpif *dpif;
131
185
struct ovs_refcount ref_cnt;
132
186
atomic_flag destroyed;
133
187
134
/* Flows.
135
*
136
* Readers of 'cls' and 'flow_table' must take a 'cls->rwlock' read lock.
137
*
138
* Writers of 'cls' and 'flow_table' must take the 'flow_mutex' and then
139
* the 'cls->rwlock' write lock. (The outer 'flow_mutex' allows writers to
140
* atomically perform multiple operations on 'cls' and 'flow_table'.)
141
*/
142
struct ovs_mutex flow_mutex;
143
struct classifier cls; /* Classifier. Protected by cls.rwlock. */
144
struct hmap flow_table OVS_GUARDED; /* Flow table. */
145
146
/* Queues.
147
*
148
* 'queue_rwlock' protects the modification of 'handler_queues' and
149
* 'n_handlers'. The queue elements are protected by its
150
* 'handler_queues''s mutex. */
151
struct fat_rwlock queue_rwlock;
152
struct dp_netdev_queue *handler_queues;
153
uint32_t n_handlers;
154
155
/* Statistics.
156
*
157
* ovsthread_stats is internally synchronized. */
158
struct ovsthread_stats stats; /* Contains 'struct dp_netdev_stats *'. */
159
160
188
/* Ports.
161
189
*
162
* Any lookup into 'ports' or any access to the dp_netdev_ports found
163
* through 'ports' requires taking 'port_rwlock'. */
164
struct ovs_rwlock port_rwlock;
165
struct hmap ports OVS_GUARDED;
190
* Protected by RCU. Take the mutex to add or remove ports. */
191
struct ovs_mutex port_mutex;
192
struct cmap ports;
166
193
struct seq *port_seq; /* Incremented whenever a port changes. */
167
194
168
/* Forwarding threads. */
169
struct latch exit_latch;
170
struct pmd_thread *pmd_threads;
171
size_t n_pmd_threads;
172
int pmd_count;
195
/* Protects access to ofproto-dpif-upcall interface during revalidator
196
* thread synchronization. */
197
struct fat_rwlock upcall_rwlock;
198
upcall_callback *upcall_cb; /* Callback function for executing upcalls. */
199
void *upcall_aux;
200
201
/* Stores all 'struct dp_netdev_pmd_thread's. */
202
struct cmap poll_threads;
203
204
/* Protects the access of the 'struct dp_netdev_pmd_thread'
205
* instance for non-pmd thread. */
206
struct ovs_mutex non_pmd_mutex;
207
208
/* Each pmd thread will store its pointer to
209
* 'struct dp_netdev_pmd_thread' in 'per_pmd_key'. */
210
ovsthread_key_t per_pmd_key;
211
212
/* Number of rx queues for each dpdk interface and the cpu mask
213
* for pin of pmd threads. */
214
size_t n_dpdk_rxqs;
215
char *pmd_cmask;
216
uint64_t last_tnl_conf_seq;
173
217
};
174
218
175
219
static struct dp_netdev_port *dp_netdev_lookup_port(const struct dp_netdev *dp,
176
odp_port_t)
177
OVS_REQ_RDLOCK(dp->port_rwlock);
220
odp_port_t);
178
221
179
222
enum dp_stat_type {
180
DP_STAT_HIT, /* Packets that matched in the flow table. */
223
DP_STAT_EXACT_HIT, /* Packets that had an exact match (emc). */
224
DP_STAT_MASKED_HIT, /* Packets that matched in the flow table. */
181
225
DP_STAT_MISS, /* Packets that did not match. */
182
226
DP_STAT_LOST, /* Packets not passed up to the client. */
183
227
DP_N_STATS
184
228
};
185
229
186
/* Contained by struct dp_netdev's 'stats' member. */
187
struct dp_netdev_stats {
188
struct ovs_mutex mutex; /* Protects 'n'. */
189
190
/* Indexed by DP_STAT_*, protected by 'mutex'. */
191
unsigned long long int n[DP_N_STATS] OVS_GUARDED;
230
enum pmd_cycles_counter_type {
231
PMD_CYCLES_POLLING, /* Cycles spent polling NICs. */
232
PMD_CYCLES_PROCESSING, /* Cycles spent processing packets */
233
PMD_N_CYCLES
192
234
};
193
235
194
195
236
/* A port in a netdev-based datapath. */
196
237
struct dp_netdev_port {
197
struct hmap_node node; /* Node in dp_netdev's 'ports'. */
198
odp_port_t port_no;
238
struct pkt_metadata md;
199
239
struct netdev *netdev;
240
struct cmap_node node; /* Node in dp_netdev's 'ports'. */
200
241
struct netdev_saved_flags *sf;
201
242
struct netdev_rxq **rxq;
202
243
struct ovs_refcount ref_cnt;
203
244
char *type; /* Port type as requested by user. */
204
245
};
205
246
206
/* A flow in dp_netdev's 'flow_table'.
247
/* Contained by struct dp_netdev_flow's 'stats' member. */
248
struct dp_netdev_flow_stats {
249
atomic_llong used; /* Last used time, in monotonic msecs. */
250
atomic_ullong packet_count; /* Number of packets matched. */
251
atomic_ullong byte_count; /* Number of bytes matched. */
252
atomic_uint16_t tcp_flags; /* Bitwise-OR of seen tcp_flags values. */
253
};
254
255
/* A flow in 'dp_netdev_pmd_thread's 'flow_table'.
207
256
*
208
257
*
209
258
* Thread-safety
210
259
* =============
211
260
*
212
261
* Except near the beginning or ending of its lifespan, rule 'rule' belongs to
213
* its dp_netdev's classifier. The text below calls this classifier 'cls'.
262
* its pmd thread's classifier. The text below calls this classifier 'cls'.
214
263
*
215
264
* Motivation
216
265
* ----------
229
278
* Rules
230
279
* -----
231
280
*
232
* A flow 'flow' may be accessed without a risk of being freed by code that
233
* holds a read-lock or write-lock on 'cls->rwlock' or that owns a reference to
234
* 'flow->ref_cnt' (or both). Code that needs to hold onto a flow for a while
235
* should take 'cls->rwlock', find the flow it needs, increment 'flow->ref_cnt'
236
* with dpif_netdev_flow_ref(), and drop 'cls->rwlock'.
281
* A flow 'flow' may be accessed without a risk of being freed during an RCU
282
* grace period. Code that needs to hold onto a flow for a while
283
* should try incrementing 'flow->ref_cnt' with dp_netdev_flow_ref().
237
284
*
238
285
* 'flow->ref_cnt' protects 'flow' from being freed. It doesn't protect the
239
* flow from being deleted from 'cls' (that's 'cls->rwlock') and it doesn't
240
* protect members of 'flow' from modification.
286
* flow from being deleted from 'cls' and it doesn't protect members of 'flow'
287
* from modification.
241
288
*
242
289
* Some members, marked 'const', are immutable. Accessing other members
243
290
* requires synchronization, as noted in more detail below.
244
291
*/
245
292
struct dp_netdev_flow {
246
/* Packet classification. */
247
const struct cls_rule cr; /* In owning dp_netdev's 'cls'. */
248
293
const struct flow flow; /* Unmasked flow that created this entry. */
249
294
/* Hash table index by unmasked flow. */
250
const struct hmap_node node; /* In owning dp_netdev's 'flow_table'. */
251
const struct flow flow; /* The flow that created this entry. */
252
253
/* Statistics.
254
*
255
* Reading or writing these members requires 'mutex'. */
256
struct ovsthread_stats stats; /* Contains "struct dp_netdev_flow_stats". */
295
const struct cmap_node node; /* In owning dp_netdev_pmd_thread's */
296
/* 'flow_table'. */
297
const ovs_u128 ufid; /* Unique flow identifier. */
298
const unsigned pmd_id; /* The 'core_id' of pmd thread owning this */
299
/* flow. */
300
301
/* Number of references.
302
* The classifier owns one reference.
303
* Any thread trying to keep a rule from being freed should hold its own
304
* reference. */
305
struct ovs_refcount ref_cnt;
306
307
bool dead;
308
309
/* Statistics. */
310
struct dp_netdev_flow_stats stats;
257
311
258
312
/* Actions. */
259
313
OVSRCU_TYPE(struct dp_netdev_actions *) actions;
260
};
261
262
static void dp_netdev_flow_free(struct dp_netdev_flow *);
263
264
/* Contained by struct dp_netdev_flow's 'stats' member. */
265
struct dp_netdev_flow_stats {
266
struct ovs_mutex mutex; /* Guards all the other members. */
267
268
long long int used OVS_GUARDED; /* Last used time, in monotonic msecs. */
269
long long int packet_count OVS_GUARDED; /* Number of packets matched. */
270
long long int byte_count OVS_GUARDED; /* Number of bytes matched. */
271
uint16_t tcp_flags OVS_GUARDED; /* Bitwise-OR of seen tcp_flags values. */
272
};
314
315
/* While processing a group of input packets, the datapath uses the next
316
* member to store a pointer to the output batch for the flow. It is
317
* reset after the batch has been sent out (See dp_netdev_queue_batches(),
318
* packet_batch_init() and packet_batch_execute()). */
319
struct packet_batch *batch;
320
321
/* Packet classification. */
322
struct dpcls_rule cr; /* In owning dp_netdev's 'cls'. */
323
/* 'cr' must be the last member. */
324
};
325
326
static void dp_netdev_flow_unref(struct dp_netdev_flow *);
327
static bool dp_netdev_flow_ref(struct dp_netdev_flow *);
328
static int dpif_netdev_flow_from_nlattrs(const struct nlattr *, uint32_t,
329
struct flow *);
273
330
274
331
/* A set of datapath actions within a "struct dp_netdev_flow".
275
332
*
281
338
struct dp_netdev_actions {
282
339
/* These members are immutable: they do not change during the struct's
283
340
* lifetime. */
284
struct nlattr *actions; /* Sequence of OVS_ACTION_ATTR_* attributes. */
285
341
unsigned int size; /* Size of 'actions', in bytes. */
342
struct nlattr actions[]; /* Sequence of OVS_ACTION_ATTR_* attributes. */
286
343
};
287
344
288
345
struct dp_netdev_actions *dp_netdev_actions_create(const struct nlattr *,
291
348
const struct dp_netdev_flow *);
292
349
static void dp_netdev_actions_free(struct dp_netdev_actions *);
293
350
351
/* Contained by struct dp_netdev_pmd_thread's 'stats' member. */
352
struct dp_netdev_pmd_stats {
353
/* Indexed by DP_STAT_*. */
354
atomic_ullong n[DP_N_STATS];
355
};
356
357
/* Contained by struct dp_netdev_pmd_thread's 'cycle' member. */
358
struct dp_netdev_pmd_cycles {
359
/* Indexed by PMD_CYCLES_*. */
360
atomic_ullong n[PMD_N_CYCLES];
361
};
362
294
363
/* PMD: Poll modes drivers. PMD accesses devices via polling to eliminate
295
364
* the performance overhead of interrupt processing. Therefore netdev can
296
365
* not implement rx-wait for these devices. dpif-netdev needs to poll
297
366
* these device to check for recv buffer. pmd-thread does polling for
298
* devices assigned to itself thread.
367
* devices assigned to itself.
299
368
*
300
369
* DPDK used PMD for accessing NIC.
301
370
*
302
* A thread that receives packets from PMD ports, looks them up in the flow
303
* table, and executes the actions it finds.
304
**/
305
struct pmd_thread {
371
* Note, instance with cpu core id NON_PMD_CORE_ID will be reserved for
372
* I/O of all non-pmd threads. There will be no actual thread created
373
* for the instance.
374
*
375
* Each struct has its own flow table and classifier. Packets received
376
* from managed ports are looked up in the corresponding pmd thread's
377
* flow table, and are executed with the found actions.
378
* */
379
struct dp_netdev_pmd_thread {
306
380
struct dp_netdev *dp;
381
struct ovs_refcount ref_cnt; /* Every reference must be refcount'ed. */
382
struct cmap_node node; /* In 'dp->poll_threads'. */
383
384
pthread_cond_t cond; /* For synchronizing pmd thread reload. */
385
struct ovs_mutex cond_mutex; /* Mutex for condition variable. */
386
387
/* Per thread exact-match cache. Note, the instance for cpu core
388
* NON_PMD_CORE_ID can be accessed by multiple threads, and thusly
389
* need to be protected (e.g. by 'dp_netdev_mutex'). All other
390
* instances will only be accessed by its own pmd thread. */
391
struct emc_cache flow_cache;
392
393
/* Classifier and Flow-Table.
394
*
395
* Writers of 'flow_table' must take the 'flow_mutex'. Corresponding
396
* changes to 'cls' must be made while still holding the 'flow_mutex'.
397
*/
398
struct ovs_mutex flow_mutex;
399
struct dpcls cls;
400
struct cmap flow_table OVS_GUARDED; /* Flow table. */
401
402
/* Statistics. */
403
struct dp_netdev_pmd_stats stats;
404
405
/* Cycles counters */
406
struct dp_netdev_pmd_cycles cycles;
407
408
/* Used to count cicles. See 'cycles_counter_end()' */
409
unsigned long long last_cycles;
410
411
struct latch exit_latch; /* For terminating the pmd thread. */
412
atomic_uint change_seq; /* For reloading pmd ports. */
307
413
pthread_t thread;
308
int id;
309
atomic_uint change_seq;
414
int index; /* Idx of this pmd thread among pmd*/
415
/* threads on same numa node. */
416
unsigned core_id; /* CPU core id of this pmd thread. */
417
int numa_id; /* numa node id of this pmd thread. */
418
int tx_qid; /* Queue id used by this pmd thread to
419
* send packets on all netdevs */
420
421
/* Only a pmd thread can write on its own 'cycles' and 'stats'.
422
* The main thread keeps 'stats_zero' and 'cycles_zero' as base
423
* values and subtracts them from 'stats' and 'cycles' before
424
* reporting to the user */
425
unsigned long long stats_zero[DP_N_STATS];
426
uint64_t cycles_zero[PMD_N_CYCLES];
310
427
};
311
428
429
#define PMD_INITIAL_SEQ 1
430
312
431
/* Interface to netdev-based datapath. */
313
432
struct dpif_netdev {
314
433
struct dpif dpif;
317
436
};
318
437
319
438
static int get_port_by_number(struct dp_netdev *dp, odp_port_t port_no,
320
struct dp_netdev_port **portp)
321
OVS_REQ_RDLOCK(dp->port_rwlock);
439
struct dp_netdev_port **portp);
322
440
static int get_port_by_name(struct dp_netdev *dp, const char *devname,
323
struct dp_netdev_port **portp)
324
OVS_REQ_RDLOCK(dp->port_rwlock);
441
struct dp_netdev_port **portp);
325
442
static void dp_netdev_free(struct dp_netdev *)
326
443
OVS_REQUIRES(dp_netdev_mutex);
327
static void dp_netdev_flow_flush(struct dp_netdev *);
328
444
static int do_add_port(struct dp_netdev *dp, const char *devname,
329
445
const char *type, odp_port_t port_no)
330
OVS_REQ_WRLOCK(dp->port_rwlock);
331
static int do_del_port(struct dp_netdev *dp, odp_port_t port_no)
332
OVS_REQ_WRLOCK(dp->port_rwlock);
333
static void dp_netdev_destroy_all_queues(struct dp_netdev *dp)
334
OVS_REQ_WRLOCK(dp->queue_rwlock);
446
OVS_REQUIRES(dp->port_mutex);
447
static void do_del_port(struct dp_netdev *dp, struct dp_netdev_port *)
448
OVS_REQUIRES(dp->port_mutex);
335
449
static int dpif_netdev_open(const struct dpif_class *, const char *name,
336
450
bool create, struct dpif **);
337
static int dp_netdev_output_userspace(struct dp_netdev *dp, struct ofpbuf *,
338
int queue_no, int type,
339
const struct miniflow *,
340
const struct nlattr *userdata);
341
static void dp_netdev_execute_actions(struct dp_netdev *dp,
342
const struct miniflow *,
343
struct ofpbuf *, bool may_steal,
344
struct pkt_metadata *,
451
static void dp_netdev_execute_actions(struct dp_netdev_pmd_thread *pmd,
452
struct dp_packet **, int c,
453
bool may_steal,
345
454
const struct nlattr *actions,
346
455
size_t actions_len);
347
static void dp_netdev_port_input(struct dp_netdev *dp, struct ofpbuf *packet,
348
struct pkt_metadata *);
349
350
static void dp_netdev_set_pmd_threads(struct dp_netdev *, int n);
456
static void dp_netdev_input(struct dp_netdev_pmd_thread *,
457
struct dp_packet **, int cnt);
458
459
static void dp_netdev_disable_upcall(struct dp_netdev *);
460
void dp_netdev_pmd_reload_done(struct dp_netdev_pmd_thread *pmd);
461
static void dp_netdev_configure_pmd(struct dp_netdev_pmd_thread *pmd,
462
struct dp_netdev *dp, int index,
463
unsigned core_id, int numa_id);
464
static void dp_netdev_destroy_pmd(struct dp_netdev_pmd_thread *pmd);
465
static void dp_netdev_set_nonpmd(struct dp_netdev *dp);
466
static struct dp_netdev_pmd_thread *dp_netdev_get_pmd(struct dp_netdev *dp,
467
unsigned core_id);
468
static struct dp_netdev_pmd_thread *
469
dp_netdev_pmd_get_next(struct dp_netdev *dp, struct cmap_position *pos);
470
static void dp_netdev_destroy_all_pmds(struct dp_netdev *dp);
471
static void dp_netdev_del_pmds_on_numa(struct dp_netdev *dp, int numa_id);
472
static void dp_netdev_set_pmds_on_numa(struct dp_netdev *dp, int numa_id);
473
static void dp_netdev_reset_pmd_threads(struct dp_netdev *dp);
474
static bool dp_netdev_pmd_try_ref(struct dp_netdev_pmd_thread *pmd);
475
static void dp_netdev_pmd_unref(struct dp_netdev_pmd_thread *pmd);
476
static void dp_netdev_pmd_flow_flush(struct dp_netdev_pmd_thread *pmd);
477
478
static inline bool emc_entry_alive(struct emc_entry *ce);
479
static void emc_clear_entry(struct emc_entry *ce);
480
481
static void
482
emc_cache_init(struct emc_cache *flow_cache)
483
{
484
int i;
485
486
BUILD_ASSERT(offsetof(struct miniflow, inline_values) == sizeof(uint64_t));
487
488
flow_cache->sweep_idx = 0;
489
for (i = 0; i < ARRAY_SIZE(flow_cache->entries); i++) {
490
flow_cache->entries[i].flow = NULL;
491
flow_cache->entries[i].key.hash = 0;
492
flow_cache->entries[i].key.len
493
= offsetof(struct miniflow, inline_values);
494
miniflow_initialize(&flow_cache->entries[i].key.mf,
495
flow_cache->entries[i].key.buf);
496
}
497
}
498
499
static void
500
emc_cache_uninit(struct emc_cache *flow_cache)
501
{
502
int i;
503
504
for (i = 0; i < ARRAY_SIZE(flow_cache->entries); i++) {
505
emc_clear_entry(&flow_cache->entries[i]);
506
}
507
}
508
509
/* Check and clear dead flow references slowly (one entry at each
510
* invocation). */
511
static void
512
emc_cache_slow_sweep(struct emc_cache *flow_cache)
513
{
514
struct emc_entry *entry = &flow_cache->entries[flow_cache->sweep_idx];
515
516
if (!emc_entry_alive(entry)) {
517
emc_clear_entry(entry);
518
}
519
flow_cache->sweep_idx = (flow_cache->sweep_idx + 1) & EM_FLOW_HASH_MASK;
520
}
351
521
352
522
static struct dpif_netdev *
353
523
dpif_netdev_cast(const struct dpif *dpif)
361
531
{
362
532
return dpif_netdev_cast(dpif)->dp;
363
533
}
364
365
static int
366
dpif_netdev_enumerate(struct sset *all_dps)
534
535
enum pmd_info_type {
536
PMD_INFO_SHOW_STATS, /* show how cpu cycles are spent */
537
PMD_INFO_CLEAR_STATS /* set the cycles count to 0 */
538
};
539
540
static void
541
pmd_info_show_stats(struct ds *reply,
542
struct dp_netdev_pmd_thread *pmd,
543
unsigned long long stats[DP_N_STATS],
544
uint64_t cycles[PMD_N_CYCLES])
545
{
546
unsigned long long total_packets = 0;
547
uint64_t total_cycles = 0;
548
int i;
549
550
/* These loops subtracts reference values ('*_zero') from the counters.
551
* Since loads and stores are relaxed, it might be possible for a '*_zero'
552
* value to be more recent than the current value we're reading from the
553
* counter. This is not a big problem, since these numbers are not
554
* supposed to be too accurate, but we should at least make sure that
555
* the result is not negative. */
556
for (i = 0; i < DP_N_STATS; i++) {
557
if (stats[i] > pmd->stats_zero[i]) {
558
stats[i] -= pmd->stats_zero[i];
559
} else {
560
stats[i] = 0;
561
}
562
563
if (i != DP_STAT_LOST) {
564
/* Lost packets are already included in DP_STAT_MISS */
565
total_packets += stats[i];
566
}
567
}
568
569
for (i = 0; i < PMD_N_CYCLES; i++) {
570
if (cycles[i] > pmd->cycles_zero[i]) {
571
cycles[i] -= pmd->cycles_zero[i];
572
} else {
573
cycles[i] = 0;
574
}
575
576
total_cycles += cycles[i];
577
}
578
579
ds_put_cstr(reply, (pmd->core_id == NON_PMD_CORE_ID)
580
? "main thread" : "pmd thread");
581
582
if (pmd->numa_id != OVS_NUMA_UNSPEC) {
583
ds_put_format(reply, " numa_id %d", pmd->numa_id);
584
}
585
if (pmd->core_id != OVS_CORE_UNSPEC && pmd->core_id != NON_PMD_CORE_ID) {
586
ds_put_format(reply, " core_id %u", pmd->core_id);
587
}
588
ds_put_cstr(reply, ":\n");
589
590
ds_put_format(reply,
591
"\temc hits:%llu\n\tmegaflow hits:%llu\n"
592
"\tmiss:%llu\n\tlost:%llu\n",
593
stats[DP_STAT_EXACT_HIT], stats[DP_STAT_MASKED_HIT],
594
stats[DP_STAT_MISS], stats[DP_STAT_LOST]);
595
596
if (total_cycles == 0) {
597
return;
598
}
599
600
ds_put_format(reply,
601
"\tpolling cycles:%"PRIu64" (%.02f%%)\n"
602
"\tprocessing cycles:%"PRIu64" (%.02f%%)\n",
603
cycles[PMD_CYCLES_POLLING],
604
cycles[PMD_CYCLES_POLLING] / (double)total_cycles * 100,
605
cycles[PMD_CYCLES_PROCESSING],
606
cycles[PMD_CYCLES_PROCESSING] / (double)total_cycles * 100);
607
608
if (total_packets == 0) {
609
return;
610
}
611
612
ds_put_format(reply,
613
"\tavg cycles per packet: %.02f (%"PRIu64"/%llu)\n",
614
total_cycles / (double)total_packets,
615
total_cycles, total_packets);
616
617
ds_put_format(reply,
618
"\tavg processing cycles per packet: "
619
"%.02f (%"PRIu64"/%llu)\n",
620
cycles[PMD_CYCLES_PROCESSING] / (double)total_packets,
621
cycles[PMD_CYCLES_PROCESSING], total_packets);
622
}
623
624
static void
625
pmd_info_clear_stats(struct ds *reply OVS_UNUSED,
626
struct dp_netdev_pmd_thread *pmd,
627
unsigned long long stats[DP_N_STATS],
628
uint64_t cycles[PMD_N_CYCLES])
629
{
630
int i;
631
632
/* We cannot write 'stats' and 'cycles' (because they're written by other
633
* threads) and we shouldn't change 'stats' (because they're used to count
634
* datapath stats, which must not be cleared here). Instead, we save the
635
* current values and subtract them from the values to be displayed in the
636
* future */
637
for (i = 0; i < DP_N_STATS; i++) {
638
pmd->stats_zero[i] = stats[i];
639
}
640
for (i = 0; i < PMD_N_CYCLES; i++) {
641
pmd->cycles_zero[i] = cycles[i];
642
}
643
}
644
645
static void
646
dpif_netdev_pmd_info(struct unixctl_conn *conn, int argc, const char *argv[],
647
void *aux)
648
{
649
struct ds reply = DS_EMPTY_INITIALIZER;
650
struct dp_netdev_pmd_thread *pmd;
651
struct dp_netdev *dp = NULL;
652
enum pmd_info_type type = *(enum pmd_info_type *) aux;
653
654
ovs_mutex_lock(&dp_netdev_mutex);
655
656
if (argc == 2) {
657
dp = shash_find_data(&dp_netdevs, argv[1]);
658
} else if (shash_count(&dp_netdevs) == 1) {
659
/* There's only one datapath */
660
dp = shash_first(&dp_netdevs)->data;
661
}
662
663
if (!dp) {
664
ovs_mutex_unlock(&dp_netdev_mutex);
665
unixctl_command_reply_error(conn,
666
"please specify an existing datapath");
667
return;
668
}
669
670
CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
671
unsigned long long stats[DP_N_STATS];
672
uint64_t cycles[PMD_N_CYCLES];
673
int i;
674
675
/* Read current stats and cycle counters */
676
for (i = 0; i < ARRAY_SIZE(stats); i++) {
677
atomic_read_relaxed(&pmd->stats.n[i], &stats[i]);
678
}
679
for (i = 0; i < ARRAY_SIZE(cycles); i++) {
680
atomic_read_relaxed(&pmd->cycles.n[i], &cycles[i]);
681
}
682
683
if (type == PMD_INFO_CLEAR_STATS) {
684
pmd_info_clear_stats(&reply, pmd, stats, cycles);
685
} else if (type == PMD_INFO_SHOW_STATS) {
686
pmd_info_show_stats(&reply, pmd, stats, cycles);
687
}
688
}
689
690
ovs_mutex_unlock(&dp_netdev_mutex);
691
692
unixctl_command_reply(conn, ds_cstr(&reply));
693
ds_destroy(&reply);
694
}
695
696
static int
697
dpif_netdev_init(void)
698
{
699
static enum pmd_info_type show_aux = PMD_INFO_SHOW_STATS,
700
clear_aux = PMD_INFO_CLEAR_STATS;
701
702
unixctl_command_register("dpif-netdev/pmd-stats-show", "[dp]",
703
0, 1, dpif_netdev_pmd_info,
704
(void *)&show_aux);
705
unixctl_command_register("dpif-netdev/pmd-stats-clear", "[dp]",
706
0, 1, dpif_netdev_pmd_info,
707
(void *)&clear_aux);
708
return 0;
709
}
710
711
static int
712
dpif_netdev_enumerate(struct sset *all_dps,
713
const struct dpif_class *dpif_class)
367
714
{
368
715
struct shash_node *node;
369
716
370
717
ovs_mutex_lock(&dp_netdev_mutex);
371
718
SHASH_FOR_EACH(node, &dp_netdevs) {
719
struct dp_netdev *dp = node->data;
720
if (dpif_class != dp->class) {
721
/* 'dp_netdevs' contains both "netdev" and "dummy" dpifs.
722
* If the class doesn't match, skip this dpif. */
723
continue;
724
}
372
725
sset_add(all_dps, node->name);
373
726
}
374
727
ovs_mutex_unlock(&dp_netdev_mutex);
410
763
* Return ODPP_NONE on failure. */
411
764
static odp_port_t
412
765
choose_port(struct dp_netdev *dp, const char *name)
413
OVS_REQ_RDLOCK(dp->port_rwlock)
766
OVS_REQUIRES(dp->port_mutex)
414
767
{
415
768
uint32_t port_no;
416
769
464
817
ovs_refcount_init(&dp->ref_cnt);
465
818
atomic_flag_clear(&dp->destroyed);
466
819
467
ovs_mutex_init(&dp->flow_mutex);
468
classifier_init(&dp->cls, NULL);
469
hmap_init(&dp->flow_table);
470
471
fat_rwlock_init(&dp->queue_rwlock);
472
473
ovsthread_stats_init(&dp->stats);
474
475
ovs_rwlock_init(&dp->port_rwlock);
476
hmap_init(&dp->ports);
820
ovs_mutex_init(&dp->port_mutex);
821
cmap_init(&dp->ports);
477
822
dp->port_seq = seq_create();
478
latch_init(&dp->exit_latch);
479
480
ovs_rwlock_wrlock(&dp->port_rwlock);
823
fat_rwlock_init(&dp->upcall_rwlock);
824
825
/* Disable upcalls by default. */
826
dp_netdev_disable_upcall(dp);
827
dp->upcall_aux = NULL;
828
dp->upcall_cb = NULL;
829
830
cmap_init(&dp->poll_threads);
831
ovs_mutex_init_recursive(&dp->non_pmd_mutex);
832
ovsthread_key_create(&dp->per_pmd_key, NULL);
833
834
dp_netdev_set_nonpmd(dp);
835
dp->n_dpdk_rxqs = NR_QUEUE;
836
837
ovs_mutex_lock(&dp->port_mutex);
481
838
error = do_add_port(dp, name, "internal", ODPP_LOCAL);
482
ovs_rwlock_unlock(&dp->port_rwlock);
839
ovs_mutex_unlock(&dp->port_mutex);
483
840
if (error) {
484
841
dp_netdev_free(dp);
485
842
return error;
486
843
}
487
844
845
dp->last_tnl_conf_seq = seq_read(tnl_conf_seq);
488
846
*dpp = dp;
489
847
return 0;
490
848
}
507
865
}
508
866
if (!error) {
509
867
*dpifp = create_dpif_netdev(dp);
868
dp->dpif = *dpifp;
510
869
}
511
870
ovs_mutex_unlock(&dp_netdev_mutex);
512
871
514
873
}
515
874
516
875
static void
517
dp_netdev_purge_queues(struct dp_netdev *dp)
518
OVS_REQ_WRLOCK(dp->queue_rwlock)
876
dp_netdev_destroy_upcall_lock(struct dp_netdev *dp)
877
OVS_NO_THREAD_SAFETY_ANALYSIS
519
878
{
520
int i;
521
522
for (i = 0; i < dp->n_handlers; i++) {
523
struct dp_netdev_queue *q = &dp->handler_queues[i];
524
525
ovs_mutex_lock(&q->mutex);
526
while (q->tail != q->head) {
527
struct dp_netdev_upcall *u = &q->upcalls[q->tail++ & QUEUE_MASK];
528
ofpbuf_uninit(&u->upcall.packet);
529
ofpbuf_uninit(&u->buf);
530
}
531
ovs_mutex_unlock(&q->mutex);
532
}
879
/* Check that upcalls are disabled, i.e. that the rwlock is taken */
880
ovs_assert(fat_rwlock_tryrdlock(&dp->upcall_rwlock));
881
882
/* Before freeing a lock we should release it */
883
fat_rwlock_unlock(&dp->upcall_rwlock);
884
fat_rwlock_destroy(&dp->upcall_rwlock);
533
885
}
534
886
535
887
/* Requires dp_netdev_mutex so that we can't get a new reference to 'dp'
538
890
dp_netdev_free(struct dp_netdev *dp)
539
891
OVS_REQUIRES(dp_netdev_mutex)
540
892
{
541
struct dp_netdev_port *port, *next;
542
struct dp_netdev_stats *bucket;
543
int i;
893
struct dp_netdev_port *port;
544
894
545
895
shash_find_and_delete(&dp_netdevs, dp->name);
546
896
547
dp_netdev_set_pmd_threads(dp, 0);
548
free(dp->pmd_threads);
549
550
dp_netdev_flow_flush(dp);
551
ovs_rwlock_wrlock(&dp->port_rwlock);
552
HMAP_FOR_EACH_SAFE (port, next, node, &dp->ports) {
553
do_del_port(dp, port->port_no);
554
}
555
ovs_rwlock_unlock(&dp->port_rwlock);
556
557
OVSTHREAD_STATS_FOR_EACH_BUCKET (bucket, i, &dp->stats) {
558
ovs_mutex_destroy(&bucket->mutex);
559
free_cacheline(bucket);
560
}
561
ovsthread_stats_destroy(&dp->stats);
562
563
fat_rwlock_wrlock(&dp->queue_rwlock);
564
dp_netdev_destroy_all_queues(dp);
565
fat_rwlock_unlock(&dp->queue_rwlock);
566
567
fat_rwlock_destroy(&dp->queue_rwlock);
568
569
classifier_destroy(&dp->cls);
570
hmap_destroy(&dp->flow_table);
571
ovs_mutex_destroy(&dp->flow_mutex);
897
dp_netdev_destroy_all_pmds(dp);
898
cmap_destroy(&dp->poll_threads);
899
ovs_mutex_destroy(&dp->non_pmd_mutex);
900
ovsthread_key_delete(dp->per_pmd_key);
901
902
ovs_mutex_lock(&dp->port_mutex);
903
CMAP_FOR_EACH (port, node, &dp->ports) {
904
do_del_port(dp, port);
905
}
906
ovs_mutex_unlock(&dp->port_mutex);
907
572
908
seq_destroy(dp->port_seq);
573
hmap_destroy(&dp->ports);
574
latch_destroy(&dp->exit_latch);
909
cmap_destroy(&dp->ports);
910
911
/* Upcalls must be disabled at this point */
912
dp_netdev_destroy_upcall_lock(dp);
913
914
free(dp->pmd_cmask);
575
915
free(CONST_CAST(char *, dp->name));
576
916
free(dp);
577
917
}
583
923
/* Take dp_netdev_mutex so that, if dp->ref_cnt falls to zero, we can't
584
924
* get a new reference to 'dp' through the 'dp_netdevs' shash. */
585
925
ovs_mutex_lock(&dp_netdev_mutex);
586
if (ovs_refcount_unref(&dp->ref_cnt) == 1) {
926
if (ovs_refcount_unref_relaxed(&dp->ref_cnt) == 1) {
587
927
dp_netdev_free(dp);
588
928
}
589
929
ovs_mutex_unlock(&dp_netdev_mutex);
605
945
struct dp_netdev *dp = get_dp_netdev(dpif);
606
946
607
947
if (!atomic_flag_test_and_set(&dp->destroyed)) {
608
if (ovs_refcount_unref(&dp->ref_cnt) == 1) {
948
if (ovs_refcount_unref_relaxed(&dp->ref_cnt) == 1) {
609
949
/* Can't happen: 'dpif' still owns a reference to 'dp'. */
610
950
OVS_NOT_REACHED();
611
951
}
614
954
return 0;
615
955
}
616
956
957
/* Add 'n' to the atomic variable 'var' non-atomically and using relaxed
958
* load/store semantics. While the increment is not atomic, the load and
959
* store operations are, making it impossible to read inconsistent values.
960
*
961
* This is used to update thread local stats counters. */
962
static void
963
non_atomic_ullong_add(atomic_ullong *var, unsigned long long n)
964
{
965
unsigned long long tmp;
966
967
atomic_read_relaxed(var, &tmp);
968
tmp += n;
969
atomic_store_relaxed(var, tmp);
970
}
971
617
972
static int
618
973
dpif_netdev_get_stats(const struct dpif *dpif, struct dpif_dp_stats *stats)
619
974
{
620
975
struct dp_netdev *dp = get_dp_netdev(dpif);
621
struct dp_netdev_stats *bucket;
622
size_t i;
623
624
fat_rwlock_rdlock(&dp->cls.rwlock);
625
stats->n_flows = hmap_count(&dp->flow_table);
626
fat_rwlock_unlock(&dp->cls.rwlock);
627
628
stats->n_hit = stats->n_missed = stats->n_lost = 0;
629
OVSTHREAD_STATS_FOR_EACH_BUCKET (bucket, i, &dp->stats) {
630
ovs_mutex_lock(&bucket->mutex);
631
stats->n_hit += bucket->n[DP_STAT_HIT];
632
stats->n_missed += bucket->n[DP_STAT_MISS];
633
stats->n_lost += bucket->n[DP_STAT_LOST];
634
ovs_mutex_unlock(&bucket->mutex);
976
struct dp_netdev_pmd_thread *pmd;
977
978
stats->n_flows = stats->n_hit = stats->n_missed = stats->n_lost = 0;
979
CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
980
unsigned long long n;
981
stats->n_flows += cmap_count(&pmd->flow_table);
982
983
atomic_read_relaxed(&pmd->stats.n[DP_STAT_MASKED_HIT], &n);
984
stats->n_hit += n;
985
atomic_read_relaxed(&pmd->stats.n[DP_STAT_EXACT_HIT], &n);
986
stats->n_hit += n;
987
atomic_read_relaxed(&pmd->stats.n[DP_STAT_MISS], &n);
988
stats->n_missed += n;
989
atomic_read_relaxed(&pmd->stats.n[DP_STAT_LOST], &n);
990
stats->n_lost += n;
635
991
}
636
992
stats->n_masks = UINT32_MAX;
637
993
stats->n_mask_hit = UINT64_MAX;
640
996
}
641
997
642
998
static void
643
dp_netdev_reload_pmd_threads(struct dp_netdev *dp)
644
{
645
int i;
646
647
for (i = 0; i < dp->n_pmd_threads; i++) {
648
struct pmd_thread *f = &dp->pmd_threads[i];
649
int id;
650
651
atomic_add(&f->change_seq, 1, &id);
652
}
999
dp_netdev_reload_pmd__(struct dp_netdev_pmd_thread *pmd)
1000
{
1001
int old_seq;
1002
1003
if (pmd->core_id == NON_PMD_CORE_ID) {
1004
return;
1005
}
1006
1007
ovs_mutex_lock(&pmd->cond_mutex);
1008
atomic_add_relaxed(&pmd->change_seq, 1, &old_seq);
1009
ovs_mutex_cond_wait(&pmd->cond, &pmd->cond_mutex);
1010
ovs_mutex_unlock(&pmd->cond_mutex);
1011
}
1012
1013
/* Causes all pmd threads to reload its tx/rx devices.
1014
* Must be called after adding/removing ports. */
1015
static void
1016
dp_netdev_reload_pmds(struct dp_netdev *dp)
1017
{
1018
struct dp_netdev_pmd_thread *pmd;
1019
1020
CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
1021
dp_netdev_reload_pmd__(pmd);
1022
}
1023
}
1024
1025
static uint32_t
1026
hash_port_no(odp_port_t port_no)
1027
{
1028
return hash_int(odp_to_u32(port_no), 0);
653
1029
}
654
1030
655
1031
static int
656
1032
do_add_port(struct dp_netdev *dp, const char *devname, const char *type,
657
1033
odp_port_t port_no)
658
OVS_REQ_WRLOCK(dp->port_rwlock)
1034
OVS_REQUIRES(dp->port_mutex)
659
1035
{
660
1036
struct netdev_saved_flags *sf;
661
1037
struct dp_netdev_port *port;
665
1041
int error;
666
1042
int i;
667
1043
668
/* XXX reject devices already in some dp_netdev. */
1044
/* Reject devices already in 'dp'. */
1045
if (!get_port_by_name(dp, devname, &port)) {
1046
return EEXIST;
1047
}
669
1048
670
1049
/* Open and validate network device. */
671
1050
open_type = dpif_netdev_port_open_type(dp->class, type);
682
1061
return EINVAL;
683
1062
}
684
1063
1064
if (netdev_is_pmd(netdev)) {
1065
int n_cores = ovs_numa_get_n_cores();
1066
1067
if (n_cores == OVS_CORE_UNSPEC) {
1068
VLOG_ERR("%s, cannot get cpu core info", devname);
1069
return ENOENT;
1070
}
1071
/* There can only be ovs_numa_get_n_cores() pmd threads,
1072
* so creates a txq for each, and one extra for the non
1073
* pmd threads. */
1074
error = netdev_set_multiq(netdev, n_cores + 1, dp->n_dpdk_rxqs);
1075
if (error && (error != EOPNOTSUPP)) {
1076
VLOG_ERR("%s, cannot set multiq", devname);
1077
return errno;
1078
}
1079
}
685
1080
port = xzalloc(sizeof *port);
686
port->port_no = port_no;
1081
port->md = PKT_METADATA_INITIALIZER(port_no);
687
1082
port->netdev = netdev;
688
1083
port->rxq = xmalloc(sizeof *port->rxq * netdev_n_rxq(netdev));
689
1084
port->type = xstrdup(type);
694
1089
VLOG_ERR("%s: cannot receive packets on this network device (%s)",
695
1090
devname, ovs_strerror(errno));
696
1091
netdev_close(netdev);
1092
free(port->type);
1093
free(port->rxq);
1094
free(port);
697
1095
return error;
698
1096
}
699
1097
}
704
1102
netdev_rxq_close(port->rxq[i]);
705
1103
}
706
1104
netdev_close(netdev);
1105
free(port->type);
707
1106
free(port->rxq);
708
1107
free(port);
709
1108
return error;
710
1109
}
711
1110
port->sf = sf;
712
1111
713
if (netdev_is_pmd(netdev)) {
714
dp->pmd_count++;
715
dp_netdev_set_pmd_threads(dp, NR_THREADS);
716
dp_netdev_reload_pmd_threads(dp);
717
}
718
1112
ovs_refcount_init(&port->ref_cnt);
1113
cmap_insert(&dp->ports, &port->node, hash_port_no(port_no));
719
1114
720
hmap_insert(&dp->ports, &port->node, hash_int(odp_to_u32(port_no), 0));
1115
if (netdev_is_pmd(netdev)) {
1116
dp_netdev_set_pmds_on_numa(dp, netdev_get_numa_id(netdev));
1117
dp_netdev_reload_pmds(dp);
1118
}
721
1119
seq_change(dp->port_seq);
722
1120
723
1121
return 0;
733
1131
odp_port_t port_no;
734
1132
int error;
735
1133
736
ovs_rwlock_wrlock(&dp->port_rwlock);
1134
ovs_mutex_lock(&dp->port_mutex);
737
1135
dpif_port = netdev_vport_get_dpif_port(netdev, namebuf, sizeof namebuf);
738
1136
if (*port_nop != ODPP_NONE) {
739
1137
port_no = *port_nop;
746
1144
*port_nop = port_no;
747
1145
error = do_add_port(dp, dpif_port, netdev_get_type(netdev), port_no);
748
1146
}
749
ovs_rwlock_unlock(&dp->port_rwlock);
1147
ovs_mutex_unlock(&dp->port_mutex);
750
1148
751
1149
return error;
752
1150
}
757
1155
struct dp_netdev *dp = get_dp_netdev(dpif);
758
1156
int error;
759
1157
760
ovs_rwlock_wrlock(&dp->port_rwlock);
761
error = port_no == ODPP_LOCAL ? EINVAL : do_del_port(dp, port_no);
762
ovs_rwlock_unlock(&dp->port_rwlock);
1158
ovs_mutex_lock(&dp->port_mutex);
1159
if (port_no == ODPP_LOCAL) {
1160
error = EINVAL;
1161
} else {
1162
struct dp_netdev_port *port;
1163
1164
error = get_port_by_number(dp, port_no, &port);
1165
if (!error) {
1166
do_del_port(dp, port);
1167
}
1168
}
1169
ovs_mutex_unlock(&dp->port_mutex);
763
1170
764
1171
return error;
765
1172
}
772
1179
773
1180
static struct dp_netdev_port *
774
1181
dp_netdev_lookup_port(const struct dp_netdev *dp, odp_port_t port_no)
775
OVS_REQ_RDLOCK(dp->port_rwlock)
776
1182
{
777
1183
struct dp_netdev_port *port;
778
1184
779
HMAP_FOR_EACH_IN_BUCKET (port, node, hash_int(odp_to_u32(port_no), 0),
780
&dp->ports) {
781
if (port->port_no == port_no) {
1185
CMAP_FOR_EACH_WITH_HASH (port, node, hash_port_no(port_no), &dp->ports) {
1186
if (port->md.in_port.odp_port == port_no) {
782
1187
return port;
783
1188
}
784
1189
}
788
1193
static int
789
1194
get_port_by_number(struct dp_netdev *dp,
790
1195
odp_port_t port_no, struct dp_netdev_port **portp)
791
OVS_REQ_RDLOCK(dp->port_rwlock)
792
1196
{
793
1197
if (!is_valid_port_number(port_no)) {
794
1198
*portp = NULL;
807
1211
}
808
1212
}
809
1213
1214
static bool
1215
port_try_ref(struct dp_netdev_port *port)
1216
{
1217
if (port) {
1218
return ovs_refcount_try_ref_rcu(&port->ref_cnt);
1219
}
1220
1221
return false;
1222
}
1223
810
1224
static void
811
1225
port_unref(struct dp_netdev_port *port)
812
1226
{
813
if (port && ovs_refcount_unref(&port->ref_cnt) == 1) {
1227
if (port && ovs_refcount_unref_relaxed(&port->ref_cnt) == 1) {
814
1228
int n_rxq = netdev_n_rxq(port->netdev);
815
1229
int i;
816
1230
829
1243
static int
830
1244
get_port_by_name(struct dp_netdev *dp,
831
1245
const char *devname, struct dp_netdev_port **portp)
832
OVS_REQ_RDLOCK(dp->port_rwlock)
1246
OVS_REQUIRES(dp->port_mutex)
833
1247
{
834
1248
struct dp_netdev_port *port;
835
1249
836
HMAP_FOR_EACH (port, node, &dp->ports) {
1250
CMAP_FOR_EACH (port, node, &dp->ports) {
837
1251
if (!strcmp(netdev_get_name(port->netdev), devname)) {
838
1252
*portp = port;
839
1253
return 0;
843
1257
}
844
1258
845
1259
static int
846
do_del_port(struct dp_netdev *dp, odp_port_t port_no)
847
OVS_REQ_WRLOCK(dp->port_rwlock)
1260
get_n_pmd_threads_on_numa(struct dp_netdev *dp, int numa_id)
1261
{
1262
struct dp_netdev_pmd_thread *pmd;
1263
int n_pmds = 0;
1264
1265
CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
1266
if (pmd->numa_id == numa_id) {
1267
n_pmds++;
1268
}
1269
}
1270
1271
return n_pmds;
1272
}
1273
1274
/* Returns 'true' if there is a port with pmd netdev and the netdev
1275
* is on numa node 'numa_id'. */
1276
static bool
1277
has_pmd_port_for_numa(struct dp_netdev *dp, int numa_id)
848
1278
{
849
1279
struct dp_netdev_port *port;
850
int error;
851
1280
852
error = get_port_by_number(dp, port_no, &port);
853
if (error) {
854
return error;
1281
CMAP_FOR_EACH (port, node, &dp->ports) {
1282
if (netdev_is_pmd(port->netdev)
1283
&& netdev_get_numa_id(port->netdev) == numa_id) {
1284
return true;
1285
}
855
1286
}
856
1287
857
hmap_remove(&dp->ports, &port->node);
1288
return false;
1289
}
1290
1291
1292
static void
1293
do_del_port(struct dp_netdev *dp, struct dp_netdev_port *port)
1294
OVS_REQUIRES(dp->port_mutex)
1295
{
1296
cmap_remove(&dp->ports, &port->node,
1297
hash_odp_port(port->md.in_port.odp_port));
858
1298
seq_change(dp->port_seq);
859
1299
if (netdev_is_pmd(port->netdev)) {
860
dp_netdev_reload_pmd_threads(dp);
1300
int numa_id = netdev_get_numa_id(port->netdev);
1301
1302
/* If there is no netdev on the numa node, deletes the pmd threads
1303
* for that numa. Else, just reloads the queues. */
1304
if (!has_pmd_port_for_numa(dp, numa_id)) {
1305
dp_netdev_del_pmds_on_numa(dp, numa_id);
1306
}
1307
dp_netdev_reload_pmds(dp);
861
1308
}
862
1309
863
1310
port_unref(port);
864
return 0;
865
1311
}
866
1312
867
1313
static void
870
1316
{
871
1317
dpif_port->name = xstrdup(netdev_get_name(port->netdev));
872
1318
dpif_port->type = xstrdup(port->type);
873
dpif_port->port_no = port->port_no;
1319
dpif_port->port_no = port->md.in_port.odp_port;
874
1320
}
875
1321
876
1322
static int
881
1327
struct dp_netdev_port *port;
882
1328
int error;
883
1329
884
ovs_rwlock_rdlock(&dp->port_rwlock);
885
1330
error = get_port_by_number(dp, port_no, &port);
886
1331
if (!error && dpif_port) {
887
1332
answer_port_query(port, dpif_port);
888
1333
}
889
ovs_rwlock_unlock(&dp->port_rwlock);
890
1334
891
1335
return error;
892
1336
}
899
1343
struct dp_netdev_port *port;
900
1344
int error;
901
1345
902
ovs_rwlock_rdlock(&dp->port_rwlock);
1346
ovs_mutex_lock(&dp->port_mutex);
903
1347
error = get_port_by_name(dp, devname, &port);
904
1348
if (!error && dpif_port) {
905
1349
answer_port_query(port, dpif_port);
906
1350
}
907
ovs_rwlock_unlock(&dp->port_rwlock);
1351
ovs_mutex_unlock(&dp->port_mutex);
908
1352
909
1353
return error;
910
1354
}
912
1356
static void
913
1357
dp_netdev_flow_free(struct dp_netdev_flow *flow)
914
1358
{
915
struct dp_netdev_flow_stats *bucket;
916
size_t i;
917
918
OVSTHREAD_STATS_FOR_EACH_BUCKET (bucket, i, &flow->stats) {
919
ovs_mutex_destroy(&bucket->mutex);
920
free_cacheline(bucket);
921
}
922
ovsthread_stats_destroy(&flow->stats);
923
924
cls_rule_destroy(CONST_CAST(struct cls_rule *, &flow->cr));
925
1359
dp_netdev_actions_free(dp_netdev_flow_get_actions(flow));
926
1360
free(flow);
927
1361
}
928
1362
929
static void
930
dp_netdev_remove_flow(struct dp_netdev *dp, struct dp_netdev_flow *flow)
931
OVS_REQ_WRLOCK(dp->cls.rwlock)
932
OVS_REQUIRES(dp->flow_mutex)
933
{
934
struct cls_rule *cr = CONST_CAST(struct cls_rule *, &flow->cr);
935
struct hmap_node *node = CONST_CAST(struct hmap_node *, &flow->node);
936
937
classifier_remove(&dp->cls, cr);
938
hmap_remove(&dp->flow_table, node);
939
ovsrcu_postpone(dp_netdev_flow_free, flow);
940
}
941
942
static void
943
dp_netdev_flow_flush(struct dp_netdev *dp)
944
{
945
struct dp_netdev_flow *netdev_flow, *next;
946
947
ovs_mutex_lock(&dp->flow_mutex);
948
fat_rwlock_wrlock(&dp->cls.rwlock);
949
HMAP_FOR_EACH_SAFE (netdev_flow, next, node, &dp->flow_table) {
950
dp_netdev_remove_flow(dp, netdev_flow);
951
}
952
fat_rwlock_unlock(&dp->cls.rwlock);
953
ovs_mutex_unlock(&dp->flow_mutex);
1363
static void dp_netdev_flow_unref(struct dp_netdev_flow *flow)
1364
{
1365
if (ovs_refcount_unref_relaxed(&flow->ref_cnt) == 1) {
1366
ovsrcu_postpone(dp_netdev_flow_free, flow);
1367
}
1368
}
1369
1370
static uint32_t
1371
dp_netdev_flow_hash(const ovs_u128 *ufid)
1372
{
1373
return ufid->u32[0];
1374
}
1375
1376
static void
1377
dp_netdev_pmd_remove_flow(struct dp_netdev_pmd_thread *pmd,
1378
struct dp_netdev_flow *flow)
1379
OVS_REQUIRES(pmd->flow_mutex)
1380
{
1381
struct cmap_node *node = CONST_CAST(struct cmap_node *, &flow->node);
1382
1383
dpcls_remove(&pmd->cls, &flow->cr);
1384
cmap_remove(&pmd->flow_table, node, dp_netdev_flow_hash(&flow->ufid));
1385
flow->dead = true;
1386
1387
dp_netdev_flow_unref(flow);
1388
}
1389
1390
static void
1391
dp_netdev_pmd_flow_flush(struct dp_netdev_pmd_thread *pmd)
1392
{
1393
struct dp_netdev_flow *netdev_flow;
1394
1395
ovs_mutex_lock(&pmd->flow_mutex);
1396
CMAP_FOR_EACH (netdev_flow, node, &pmd->flow_table) {
1397
dp_netdev_pmd_remove_flow(pmd, netdev_flow);
1398
}
1399
ovs_mutex_unlock(&pmd->flow_mutex);
954
1400
}
955
1401
956
1402
static int
957
1403
dpif_netdev_flow_flush(struct dpif *dpif)
958
1404
{
959
1405
struct dp_netdev *dp = get_dp_netdev(dpif);
960
961
dp_netdev_flow_flush(dp);
1406
struct dp_netdev_pmd_thread *pmd;
1407
1408
CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
1409
dp_netdev_pmd_flow_flush(pmd);
1410
}
1411
962
1412
return 0;
963
1413
}
964
1414
965
1415
struct dp_netdev_port_state {
966
uint32_t bucket;
967
uint32_t offset;
1416
struct cmap_position position;
968
1417
char *name;
969
1418
};
970
1419
981
1430
{
982
1431
struct dp_netdev_port_state *state = state_;
983
1432
struct dp_netdev *dp = get_dp_netdev(dpif);
984
struct hmap_node *node;
1433
struct cmap_node *node;
985
1434
int retval;
986
1435
987
ovs_rwlock_rdlock(&dp->port_rwlock);
988
node = hmap_at_position(&dp->ports, &state->bucket, &state->offset);
1436
node = cmap_next_position(&dp->ports, &state->position);
989
1437
if (node) {
990
1438
struct dp_netdev_port *port;
991
1439
995
1443
state->name = xstrdup(netdev_get_name(port->netdev));
996
1444
dpif_port->name = state->name;
997
1445
dpif_port->type = port->type;
998
dpif_port->port_no = port->port_no;
1446
dpif_port->port_no = port->md.in_port.odp_port;
999
1447
1000
1448
retval = 0;
1001
1449
} else {
1002
1450
retval = EOF;
1003
1451
}
1004
ovs_rwlock_unlock(&dp->port_rwlock);
1005
1452
1006
1453
return retval;
1007
1454
}
1042
1489
}
1043
1490
1044
1491
static struct dp_netdev_flow *
1045
dp_netdev_flow_cast(const struct cls_rule *cr)
1492
dp_netdev_flow_cast(const struct dpcls_rule *cr)
1046
1493
{
1047
1494
return cr ? CONTAINER_OF(cr, struct dp_netdev_flow, cr) : NULL;
1048
1495
}
1049
1496
1497
static bool dp_netdev_flow_ref(struct dp_netdev_flow *flow)
1498
{
1499
return ovs_refcount_try_ref_rcu(&flow->ref_cnt);
1500
}
1501
1502
/* netdev_flow_key utilities.
1503
*
1504
* netdev_flow_key is basically a miniflow. We use these functions
1505
* (netdev_flow_key_clone, netdev_flow_key_equal, ...) instead of the miniflow
1506
* functions (miniflow_clone_inline, miniflow_equal, ...), because:
1507
*
1508
* - Since we are dealing exclusively with miniflows created by
1509
* miniflow_extract(), if the map is different the miniflow is different.
1510
* Therefore we can be faster by comparing the map and the miniflow in a
1511
* single memcmp().
1512
* _ netdev_flow_key's miniflow has always inline values.
1513
* - These functions can be inlined by the compiler.
1514
*
1515
* The following assertions make sure that what we're doing with miniflow is
1516
* safe
1517
*/
1518
BUILD_ASSERT_DECL(offsetof(struct miniflow, inline_values)
1519
== sizeof(uint64_t));
1520
1521
/* Given the number of bits set in the miniflow map, returns the size of the
1522
* 'netdev_flow_key.mf' */
1523
static inline uint32_t
1524
netdev_flow_key_size(uint32_t flow_u32s)
1525
{
1526
return offsetof(struct miniflow, inline_values) +
1527
MINIFLOW_VALUES_SIZE(flow_u32s);
1528
}
1529
1530
static inline bool
1531
netdev_flow_key_equal(const struct netdev_flow_key *a,
1532
const struct netdev_flow_key *b)
1533
{
1534
/* 'b->len' may be not set yet. */
1535
return a->hash == b->hash && !memcmp(&a->mf, &b->mf, a->len);
1536
}
1537
1538
/* Used to compare 'netdev_flow_key' in the exact match cache to a miniflow.
1539
* The maps are compared bitwise, so both 'key->mf' 'mf' must have been
1540
* generated by miniflow_extract. */
1541
static inline bool
1542
netdev_flow_key_equal_mf(const struct netdev_flow_key *key,
1543
const struct miniflow *mf)
1544
{
1545
return !memcmp(&key->mf, mf, key->len);
1546
}
1547
1548
static inline void
1549
netdev_flow_key_clone(struct netdev_flow_key *dst,
1550
const struct netdev_flow_key *src)
1551
{
1552
memcpy(dst, src,
1553
offsetof(struct netdev_flow_key, mf) + src->len);
1554
}
1555
1556
/* Slow. */
1557
static void
1558
netdev_flow_key_from_flow(struct netdev_flow_key *dst,
1559
const struct flow *src)
1560
{
1561
struct dp_packet packet;
1562
uint64_t buf_stub[512 / 8];
1563
1564
miniflow_initialize(&dst->mf, dst->buf);
1565
1566
dp_packet_use_stub(&packet, buf_stub, sizeof buf_stub);
1567
pkt_metadata_from_flow(&packet.md, src);
1568
flow_compose(&packet, src);
1569
miniflow_extract(&packet, &dst->mf);
1570
dp_packet_uninit(&packet);
1571
1572
dst->len = netdev_flow_key_size(count_1bits(dst->mf.map));
1573
dst->hash = 0; /* Not computed yet. */
1574
}
1575
1576
/* Initialize a netdev_flow_key 'mask' from 'match'. */
1577
static inline void
1578
netdev_flow_mask_init(struct netdev_flow_key *mask,
1579
const struct match *match)
1580
{
1581
const uint64_t *mask_u64 = (const uint64_t *) &match->wc.masks;
1582
uint64_t *dst = mask->mf.inline_values;
1583
uint64_t map, mask_map = 0;
1584
uint32_t hash = 0;
1585
int n;
1586
1587
/* Only check masks that make sense for the flow. */
1588
map = flow_wc_map(&match->flow);
1589
1590
while (map) {
1591
uint64_t rm1bit = rightmost_1bit(map);
1592
int i = raw_ctz(map);
1593
1594
if (mask_u64[i]) {
1595
mask_map |= rm1bit;
1596
*dst++ = mask_u64[i];
1597
hash = hash_add64(hash, mask_u64[i]);
1598
}
1599
map -= rm1bit;
1600
}
1601
1602
mask->mf.values_inline = true;
1603
mask->mf.map = mask_map;
1604
1605
hash = hash_add64(hash, mask_map);
1606
1607
n = dst - mask->mf.inline_values;
1608
1609
mask->hash = hash_finish(hash, n * 8);
1610
mask->len = netdev_flow_key_size(n);
1611
}
1612
1613
/* Initializes 'dst' as a copy of 'src' masked with 'mask'. */
1614
static inline void
1615
netdev_flow_key_init_masked(struct netdev_flow_key *dst,
1616
const struct flow *flow,
1617
const struct netdev_flow_key *mask)
1618
{
1619
uint64_t *dst_u64 = dst->mf.inline_values;
1620
const uint64_t *mask_u64 = mask->mf.inline_values;
1621
uint32_t hash = 0;
1622
uint64_t value;
1623
1624
dst->len = mask->len;
1625
dst->mf.values_inline = true;
1626
dst->mf.map = mask->mf.map;
1627
1628
FLOW_FOR_EACH_IN_MAP(value, flow, mask->mf.map) {
1629
*dst_u64 = value & *mask_u64++;
1630
hash = hash_add64(hash, *dst_u64++);
1631
}
1632
dst->hash = hash_finish(hash, (dst_u64 - dst->mf.inline_values) * 8);
1633
}
1634
1635
/* Iterate through all netdev_flow_key u64 values specified by 'MAP' */
1636
#define NETDEV_FLOW_KEY_FOR_EACH_IN_MAP(VALUE, KEY, MAP) \
1637
for (struct mf_for_each_in_map_aux aux__ \
1638
= { (KEY)->mf.inline_values, (KEY)->mf.map, MAP }; \
1639
mf_get_next_in_map(&aux__, &(VALUE)); \
1640
)
1641
1642
/* Returns a hash value for the bits of 'key' where there are 1-bits in
1643
* 'mask'. */
1644
static inline uint32_t
1645
netdev_flow_key_hash_in_mask(const struct netdev_flow_key *key,
1646
const struct netdev_flow_key *mask)
1647
{
1648
const uint64_t *p = mask->mf.inline_values;
1649
uint32_t hash = 0;
1650
uint64_t key_u64;
1651
1652
NETDEV_FLOW_KEY_FOR_EACH_IN_MAP(key_u64, key, mask->mf.map) {
1653
hash = hash_add64(hash, key_u64 & *p++);
1654
}
1655
1656
return hash_finish(hash, (p - mask->mf.inline_values) * 8);
1657
}
1658
1659
static inline bool
1660
emc_entry_alive(struct emc_entry *ce)
1661
{
1662
return ce->flow && !ce->flow->dead;
1663
}
1664
1665
static void
1666
emc_clear_entry(struct emc_entry *ce)
1667
{
1668
if (ce->flow) {
1669
dp_netdev_flow_unref(ce->flow);
1670
ce->flow = NULL;
1671
}
1672
}
1673
1674
static inline void
1675
emc_change_entry(struct emc_entry *ce, struct dp_netdev_flow *flow,
1676
const struct netdev_flow_key *key)
1677
{
1678
if (ce->flow != flow) {
1679
if (ce->flow) {
1680
dp_netdev_flow_unref(ce->flow);
1681
}
1682
1683
if (dp_netdev_flow_ref(flow)) {
1684
ce->flow = flow;
1685
} else {
1686
ce->flow = NULL;
1687
}
1688
}
1689
if (key) {
1690
netdev_flow_key_clone(&ce->key, key);
1691
}
1692
}
1693
1694
static inline void
1695
emc_insert(struct emc_cache *cache, const struct netdev_flow_key *key,
1696
struct dp_netdev_flow *flow)
1697
{
1698
struct emc_entry *to_be_replaced = NULL;
1699
struct emc_entry *current_entry;
1700
1701
EMC_FOR_EACH_POS_WITH_HASH(cache, current_entry, key->hash) {
1702
if (netdev_flow_key_equal(¤t_entry->key, key)) {
1703
/* We found the entry with the 'mf' miniflow */
1704
emc_change_entry(current_entry, flow, NULL);
1705
return;
1706
}
1707
1708
/* Replacement policy: put the flow in an empty (not alive) entry, or
1709
* in the first entry where it can be */
1710
if (!to_be_replaced
1711
|| (emc_entry_alive(to_be_replaced)
1712
&& !emc_entry_alive(current_entry))
1713
|| current_entry->key.hash < to_be_replaced->key.hash) {
1714
to_be_replaced = current_entry;
1715
}
1716
}
1717
/* We didn't find the miniflow in the cache.
1718
* The 'to_be_replaced' entry is where the new flow will be stored */
1719
1720
emc_change_entry(to_be_replaced, flow, key);
1721
}
1722
1723
static inline struct dp_netdev_flow *
1724
emc_lookup(struct emc_cache *cache, const struct netdev_flow_key *key)
1725
{
1726
struct emc_entry *current_entry;
1727
1728
EMC_FOR_EACH_POS_WITH_HASH(cache, current_entry, key->hash) {
1729
if (current_entry->key.hash == key->hash
1730
&& emc_entry_alive(current_entry)
1731
&& netdev_flow_key_equal_mf(¤t_entry->key, &key->mf)) {
1732
1733
/* We found the entry with the 'key->mf' miniflow */
1734
return current_entry->flow;
1735
}
1736
}
1737
1738
return NULL;
1739
}
1740
1050
1741
static struct dp_netdev_flow *
1051
dp_netdev_lookup_flow(const struct dp_netdev *dp, const struct miniflow *key)
1052
OVS_EXCLUDED(dp->cls.rwlock)
1742
dp_netdev_pmd_lookup_flow(const struct dp_netdev_pmd_thread *pmd,
1743
const struct netdev_flow_key *key)
1053
1744
{
1054
1745
struct dp_netdev_flow *netdev_flow;
1055
struct cls_rule *rule;
1746
struct dpcls_rule *rule;
1056
1747
1057
fat_rwlock_rdlock(&dp->cls.rwlock);
1058
rule = classifier_lookup_miniflow_first(&dp->cls, key);
1748
dpcls_lookup(&pmd->cls, key, &rule, 1);
1059
1749
netdev_flow = dp_netdev_flow_cast(rule);
1060
fat_rwlock_unlock(&dp->cls.rwlock);
1061
1750
1062
1751
return netdev_flow;
1063
1752
}
1064
1753
1065
1754
static struct dp_netdev_flow *
1066
dp_netdev_find_flow(const struct dp_netdev *dp, const struct flow *flow)
1067
OVS_REQ_RDLOCK(dp->cls.rwlock)
1755
dp_netdev_pmd_find_flow(const struct dp_netdev_pmd_thread *pmd,
1756
const ovs_u128 *ufidp, const struct nlattr *key,
1757
size_t key_len)
1068
1758
{
1069
1759
struct dp_netdev_flow *netdev_flow;
1070
1071
HMAP_FOR_EACH_WITH_HASH (netdev_flow, node, flow_hash(flow, 0),
1072
&dp->flow_table) {
1073
if (flow_equal(&netdev_flow->flow, flow)) {
1074
return netdev_flow;
1760
struct flow flow;
1761
ovs_u128 ufid;
1762
1763
/* If a UFID is not provided, determine one based on the key. */
1764
if (!ufidp && key && key_len
1765
&& !dpif_netdev_flow_from_nlattrs(key, key_len, &flow)) {
1766
dpif_flow_hash(pmd->dp->dpif, &flow, sizeof flow, &ufid);
1767
ufidp = &ufid;
1768
}
1769
1770
if (ufidp) {
1771
CMAP_FOR_EACH_WITH_HASH (netdev_flow, node, dp_netdev_flow_hash(ufidp),
1772
&pmd->flow_table) {
1773
if (ovs_u128_equals(&netdev_flow->ufid, ufidp)) {
1774
return netdev_flow;
1775
}
1075
1776
}
1076
1777
}
1077
1778
1079
1780
}
1080
1781
1081
1782
static void
1082
get_dpif_flow_stats(struct dp_netdev_flow *netdev_flow,
1783
get_dpif_flow_stats(const struct dp_netdev_flow *netdev_flow_,
1083
1784
struct dpif_flow_stats *stats)
1084
1785
{
1085
struct dp_netdev_flow_stats *bucket;
1086
size_t i;
1087
1088
memset(stats, 0, sizeof *stats);
1089
OVSTHREAD_STATS_FOR_EACH_BUCKET (bucket, i, &netdev_flow->stats) {
1090
ovs_mutex_lock(&bucket->mutex);
1091
stats->n_packets += bucket->packet_count;
1092
stats->n_bytes += bucket->byte_count;
1093
stats->used = MAX(stats->used, bucket->used);
1094
stats->tcp_flags |= bucket->tcp_flags;
1095
ovs_mutex_unlock(&bucket->mutex);
1786
struct dp_netdev_flow *netdev_flow;
1787
unsigned long long n;
1788
long long used;
1789
uint16_t flags;
1790
1791
netdev_flow = CONST_CAST(struct dp_netdev_flow *, netdev_flow_);
1792
1793
atomic_read_relaxed(&netdev_flow->stats.packet_count, &n);
1794
stats->n_packets = n;
1795
atomic_read_relaxed(&netdev_flow->stats.byte_count, &n);
1796
stats->n_bytes = n;
1797
atomic_read_relaxed(&netdev_flow->stats.used, &used);
1798
stats->used = used;
1799
atomic_read_relaxed(&netdev_flow->stats.tcp_flags, &flags);
1800
stats->tcp_flags = flags;
1801
}
1802
1803
/* Converts to the dpif_flow format, using 'key_buf' and 'mask_buf' for
1804
* storing the netlink-formatted key/mask. 'key_buf' may be the same as
1805
* 'mask_buf'. Actions will be returned without copying, by relying on RCU to
1806
* protect them. */
1807
static void
1808
dp_netdev_flow_to_dpif_flow(const struct dp_netdev_flow *netdev_flow,
1809
struct ofpbuf *key_buf, struct ofpbuf *mask_buf,
1810
struct dpif_flow *flow, bool terse)
1811
{
1812
if (terse) {
1813
memset(flow, 0, sizeof *flow);
1814
} else {
1815
struct flow_wildcards wc;
1816
struct dp_netdev_actions *actions;
1817
size_t offset;
1818
1819
miniflow_expand(&netdev_flow->cr.mask->mf, &wc.masks);
1820
1821
/* Key */
1822
offset = key_buf->size;
1823
flow->key = ofpbuf_tail(key_buf);
1824
odp_flow_key_from_flow(key_buf, &netdev_flow->flow, &wc.masks,
1825
netdev_flow->flow.in_port.odp_port, true);
1826
flow->key_len = key_buf->size - offset;
1827
1828
/* Mask */
1829
offset = mask_buf->size;
1830
flow->mask = ofpbuf_tail(mask_buf);
1831
odp_flow_key_from_mask(mask_buf, &wc.masks, &netdev_flow->flow,
1832
odp_to_u32(wc.masks.in_port.odp_port),
1833
SIZE_MAX, true);
1834
flow->mask_len = mask_buf->size - offset;
1835
1836
/* Actions */
1837
actions = dp_netdev_flow_get_actions(netdev_flow);
1838
flow->actions = actions->actions;
1839
flow->actions_len = actions->size;
1096
1840
}
1841
1842
flow->ufid = netdev_flow->ufid;
1843
flow->ufid_present = true;
1844
flow->pmd_id = netdev_flow->pmd_id;
1845
get_dpif_flow_stats(netdev_flow, &flow->stats);
1097
1846
}
1098
1847
1099
1848
static int
1189
1938
}
1190
1939
1191
1940
static int
1192
dpif_netdev_flow_get(const struct dpif *dpif,
1193
const struct nlattr *nl_key, size_t nl_key_len,
1194
struct ofpbuf **bufp,
1195
struct nlattr **maskp, size_t *mask_len,
1196
struct nlattr **actionsp, size_t *actions_len,
1197
struct dpif_flow_stats *stats)
1941
dpif_netdev_flow_get(const struct dpif *dpif, const struct dpif_flow_get *get)
1198
1942
{
1199
1943
struct dp_netdev *dp = get_dp_netdev(dpif);
1200
1944
struct dp_netdev_flow *netdev_flow;
1201
struct flow key;
1202
int error;
1945
struct dp_netdev_pmd_thread *pmd;
1946
unsigned pmd_id = get->pmd_id == PMD_ID_NULL
1947
? NON_PMD_CORE_ID : get->pmd_id;
1948
int error = 0;
1203
1949
1204
error = dpif_netdev_flow_from_nlattrs(nl_key, nl_key_len, &key);
1205
if (error) {
1206
return error;
1950
pmd = dp_netdev_get_pmd(dp, pmd_id);
1951
if (!pmd) {
1952
return EINVAL;
1207
1953
}
1208
1954
1209
fat_rwlock_rdlock(&dp->cls.rwlock);
1210
netdev_flow = dp_netdev_find_flow(dp, &key);
1211
fat_rwlock_unlock(&dp->cls.rwlock);
1212
1955
netdev_flow = dp_netdev_pmd_find_flow(pmd, get->ufid, get->key,
1956
get->key_len);
1213
1957
if (netdev_flow) {
1214
if (stats) {
1215
get_dpif_flow_stats(netdev_flow, stats);
1216
}
1217
1218
if (maskp || actionsp) {
1219
struct dp_netdev_actions *actions;
1220
size_t len = 0;
1221
1222
actions = dp_netdev_flow_get_actions(netdev_flow);
1223
len += maskp ? sizeof(struct odputil_keybuf) : 0;
1224
len += actionsp ? actions->size : 0;
1225
1226
*bufp = ofpbuf_new(len);
1227
if (maskp) {
1228
struct flow_wildcards wc;
1229
1230
minimask_expand(&netdev_flow->cr.match.mask, &wc);
1231
odp_flow_key_from_mask(*bufp, &wc.masks, &netdev_flow->flow,
1232
odp_to_u32(wc.masks.in_port.odp_port),
1233
SIZE_MAX);
1234
*maskp = ofpbuf_data(*bufp);
1235
*mask_len = ofpbuf_size(*bufp);
1236
}
1237
if (actionsp) {
1238
struct dp_netdev_actions *actions;
1239
1240
actions = dp_netdev_flow_get_actions(netdev_flow);
1241
*actionsp = ofpbuf_put(*bufp, actions->actions, actions->size);
1242
*actions_len = actions->size;
1243
}
1244
}
1245
} else {
1958
dp_netdev_flow_to_dpif_flow(netdev_flow, get->buffer, get->buffer,
1959
get->flow, false);
1960
} else {
1246
1961
error = ENOENT;
1247
1962
}
1963
dp_netdev_pmd_unref(pmd);
1964
1248
1965
1249
1966
return error;
1250
1967
}
1251
1968
1969
static struct dp_netdev_flow *
1970
dp_netdev_flow_add(struct dp_netdev_pmd_thread *pmd,
1971
struct match *match, const ovs_u128 *ufid,
1972
const struct nlattr *actions, size_t actions_len)
1973
OVS_REQUIRES(pmd->flow_mutex)
1974
{
1975
struct dp_netdev_flow *flow;
1976
struct netdev_flow_key mask;
1977
1978
netdev_flow_mask_init(&mask, match);
1979
/* Make sure wc does not have metadata. */
1980
ovs_assert(!(mask.mf.map & (MINIFLOW_MAP(metadata) | MINIFLOW_MAP(regs))));
1981
1982
/* Do not allocate extra space. */
1983
flow = xmalloc(sizeof *flow - sizeof flow->cr.flow.mf + mask.len);
1984
memset(&flow->stats, 0, sizeof flow->stats);
1985
flow->dead = false;
1986
flow->batch = NULL;
1987
*CONST_CAST(unsigned *, &flow->pmd_id) = pmd->core_id;
1988
*CONST_CAST(struct flow *, &flow->flow) = match->flow;
1989
*CONST_CAST(ovs_u128 *, &flow->ufid) = *ufid;
1990
ovs_refcount_init(&flow->ref_cnt);
1991
ovsrcu_set(&flow->actions, dp_netdev_actions_create(actions, actions_len));
1992
1993
netdev_flow_key_init_masked(&flow->cr.flow, &match->flow, &mask);
1994
dpcls_insert(&pmd->cls, &flow->cr, &mask);
1995
1996
cmap_insert(&pmd->flow_table, CONST_CAST(struct cmap_node *, &flow->node),
1997
dp_netdev_flow_hash(&flow->ufid));
1998
1999
if (OVS_UNLIKELY(VLOG_IS_DBG_ENABLED())) {
2000
struct match match;
2001
struct ds ds = DS_EMPTY_INITIALIZER;
2002
2003
match.flow = flow->flow;
2004
miniflow_expand(&flow->cr.mask->mf, &match.wc.masks);
2005
2006
ds_put_cstr(&ds, "flow_add: ");
2007
odp_format_ufid(ufid, &ds);
2008
ds_put_cstr(&ds, " ");
2009
match_format(&match, &ds, OFP_DEFAULT_PRIORITY);
2010
ds_put_cstr(&ds, ", actions:");
2011
format_odp_actions(&ds, actions, actions_len);
2012
2013
VLOG_DBG_RL(&upcall_rl, "%s", ds_cstr(&ds));
2014
2015
ds_destroy(&ds);
2016
}
2017
2018
return flow;
2019
}
2020
1252
2021
static int
1253
dp_netdev_flow_add(struct dp_netdev *dp, const struct flow *flow,
1254
const struct flow_wildcards *wc,
1255
const struct nlattr *actions,
1256
size_t actions_len)
1257
OVS_REQUIRES(dp->flow_mutex)
2022
dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put)
1258
2023
{
2024
struct dp_netdev *dp = get_dp_netdev(dpif);
1259
2025
struct dp_netdev_flow *netdev_flow;
2026
struct netdev_flow_key key;
2027
struct dp_netdev_pmd_thread *pmd;
1260
2028
struct match match;
1261
1262
netdev_flow = xzalloc(sizeof *netdev_flow);
1263
*CONST_CAST(struct flow *, &netdev_flow->flow) = *flow;
1264
1265
ovsthread_stats_init(&netdev_flow->stats);
1266
1267
ovsrcu_set(&netdev_flow->actions,
1268
dp_netdev_actions_create(actions, actions_len));
1269
1270
match_init(&match, flow, wc);
1271
cls_rule_init(CONST_CAST(struct cls_rule *, &netdev_flow->cr),
1272
&match, NETDEV_RULE_PRIORITY);
1273
fat_rwlock_wrlock(&dp->cls.rwlock);
1274
classifier_insert(&dp->cls,
1275
CONST_CAST(struct cls_rule *, &netdev_flow->cr));
1276
hmap_insert(&dp->flow_table,
1277
CONST_CAST(struct hmap_node *, &netdev_flow->node),
1278
flow_hash(flow, 0));
1279
fat_rwlock_unlock(&dp->cls.rwlock);
1280
1281
return 0;
1282
}
1283
1284
static void
1285
clear_stats(struct dp_netdev_flow *netdev_flow)
1286
{
1287
struct dp_netdev_flow_stats *bucket;
1288
size_t i;
1289
1290
OVSTHREAD_STATS_FOR_EACH_BUCKET (bucket, i, &netdev_flow->stats) {
1291
ovs_mutex_lock(&bucket->mutex);
1292
bucket->used = 0;
1293
bucket->packet_count = 0;
1294
bucket->byte_count = 0;
1295
bucket->tcp_flags = 0;
1296
ovs_mutex_unlock(&bucket->mutex);
1297
}
1298
}
1299
1300
static int
1301
dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put)
1302
{
1303
struct dp_netdev *dp = get_dp_netdev(dpif);
1304
struct dp_netdev_flow *netdev_flow;
1305
struct flow flow;
1306
struct miniflow miniflow;
1307
struct flow_wildcards wc;
2029
ovs_u128 ufid;
2030
unsigned pmd_id = put->pmd_id == PMD_ID_NULL
2031
? NON_PMD_CORE_ID : put->pmd_id;
1308
2032
int error;
1309
2033
1310
error = dpif_netdev_flow_from_nlattrs(put->key, put->key_len, &flow);
2034
error = dpif_netdev_flow_from_nlattrs(put->key, put->key_len, &match.flow);
1311
2035
if (error) {
1312
2036
return error;
1313
2037
}
1314
2038
error = dpif_netdev_mask_from_nlattrs(put->key, put->key_len,
1315
2039
put->mask, put->mask_len,
1316
&flow, &wc.masks);
2040
&match.flow, &match.wc.masks);
1317
2041
if (error) {
1318
2042
return error;
1319
2043
}
1320
miniflow_init(&miniflow, &flow);
1321
1322
ovs_mutex_lock(&dp->flow_mutex);
1323
netdev_flow = dp_netdev_lookup_flow(dp, &miniflow);
2044
2045
pmd = dp_netdev_get_pmd(dp, pmd_id);
2046
if (!pmd) {
2047
return EINVAL;
2048
}
2049
2050
/* Must produce a netdev_flow_key for lookup.
2051
* This interface is no longer performance critical, since it is not used
2052
* for upcall processing any more. */
2053
netdev_flow_key_from_flow(&key, &match.flow);
2054
2055
if (put->ufid) {
2056
ufid = *put->ufid;
2057
} else {
2058
dpif_flow_hash(dpif, &match.flow, sizeof match.flow, &ufid);
2059
}
2060
2061
ovs_mutex_lock(&pmd->flow_mutex);
2062
netdev_flow = dp_netdev_pmd_lookup_flow(pmd, &key);
1324
2063
if (!netdev_flow) {
1325
2064
if (put->flags & DPIF_FP_CREATE) {
1326
if (hmap_count(&dp->flow_table) < MAX_FLOWS) {
2065
if (cmap_count(&pmd->flow_table) < MAX_FLOWS) {
1327
2066
if (put->stats) {
1328
2067
memset(put->stats, 0, sizeof *put->stats);
1329
2068
}
1330
error = dp_netdev_flow_add(dp, &flow, &wc, put->actions,
1331
put->actions_len);
2069
dp_netdev_flow_add(pmd, &match, &ufid, put->actions,
2070
put->actions_len);
2071
error = 0;
1332
2072
} else {
1333
2073
error = EFBIG;
1334
2074
}
1337
2077
}
1338
2078
} else {
1339
2079
if (put->flags & DPIF_FP_MODIFY
1340
&& flow_equal(&flow, &netdev_flow->flow)) {
2080
&& flow_equal(&match.flow, &netdev_flow->flow)) {
1341
2081
struct dp_netdev_actions *new_actions;
1342
2082
struct dp_netdev_actions *old_actions;
1343
2083
1351
2091
get_dpif_flow_stats(netdev_flow, put->stats);
1352
2092
}
1353
2093
if (put->flags & DPIF_FP_ZERO_STATS) {
1354
clear_stats(netdev_flow);
2094
/* XXX: The userspace datapath uses thread local statistics
2095
* (for flows), which should be updated only by the owning
2096
* thread. Since we cannot write on stats memory here,
2097
* we choose not to support this flag. Please note:
2098
* - This feature is currently used only by dpctl commands with
2099
* option --clear.
2100
* - Should the need arise, this operation can be implemented
2101
* by keeping a base value (to be update here) for each
2102
* counter, and subtracting it before outputting the stats */
2103
error = EOPNOTSUPP;
1355
2104
}
1356
2105
1357
2106
ovsrcu_postpone(dp_netdev_actions_free, old_actions);
1362
2111
error = EINVAL;
1363
2112
}
1364
2113
}
1365
ovs_mutex_unlock(&dp->flow_mutex);
1366
miniflow_destroy(&miniflow);
2114
ovs_mutex_unlock(&pmd->flow_mutex);
2115
dp_netdev_pmd_unref(pmd);
1367
2116
1368
2117
return error;
1369
2118
}
1373
2122
{
1374
2123
struct dp_netdev *dp = get_dp_netdev(dpif);
1375
2124
struct dp_netdev_flow *netdev_flow;
1376
struct flow key;
1377
int error;
2125
struct dp_netdev_pmd_thread *pmd;
2126
unsigned pmd_id = del->pmd_id == PMD_ID_NULL
2127
? NON_PMD_CORE_ID : del->pmd_id;
2128
int error = 0;
1378
2129
1379
error = dpif_netdev_flow_from_nlattrs(del->key, del->key_len, &key);
1380
if (error) {
1381
return error;
2130
pmd = dp_netdev_get_pmd(dp, pmd_id);
2131
if (!pmd) {
2132
return EINVAL;
1382
2133
}
1383
2134
1384
ovs_mutex_lock(&dp->flow_mutex);
1385
fat_rwlock_wrlock(&dp->cls.rwlock);
1386
netdev_flow = dp_netdev_find_flow(dp, &key);
2135
ovs_mutex_lock(&pmd->flow_mutex);
2136
netdev_flow = dp_netdev_pmd_find_flow(pmd, del->ufid, del->key,
2137
del->key_len);
1387
2138
if (netdev_flow) {
1388
2139
if (del->stats) {
1389
2140
get_dpif_flow_stats(netdev_flow, del->stats);
1390
2141
}
1391
dp_netdev_remove_flow(dp, netdev_flow);
2142
dp_netdev_pmd_remove_flow(pmd, netdev_flow);
1392
2143
} else {
1393
2144
error = ENOENT;
1394
2145
}
1395
fat_rwlock_unlock(&dp->cls.rwlock);
1396
ovs_mutex_unlock(&dp->flow_mutex);
2146
ovs_mutex_unlock(&pmd->flow_mutex);
2147
dp_netdev_pmd_unref(pmd);
1397
2148
1398
2149
return error;
1399
2150
}
1400
2151
1401
struct dp_netdev_flow_state {
1402
struct odputil_keybuf keybuf;
1403
struct odputil_keybuf maskbuf;
1404
struct dpif_flow_stats stats;
1405
};
1406
1407
struct dp_netdev_flow_iter {
1408
uint32_t bucket;
1409
uint32_t offset;
2152
struct dpif_netdev_flow_dump {
2153
struct dpif_flow_dump up;
2154
struct cmap_position poll_thread_pos;
2155
struct cmap_position flow_pos;
2156
struct dp_netdev_pmd_thread *cur_pmd;
1410
2157
int status;
1411
2158
struct ovs_mutex mutex;
1412
2159
};
1413
2160
1414
static void
1415
dpif_netdev_flow_dump_state_init(void **statep)
1416
{
1417
struct dp_netdev_flow_state *state;
1418
1419
*statep = state = xmalloc(sizeof *state);
1420
}
1421
1422
static void
1423
dpif_netdev_flow_dump_state_uninit(void *state_)
1424
{
1425
struct dp_netdev_flow_state *state = state_;
1426
1427
free(state);
1428
}
1429
1430
static int
1431
dpif_netdev_flow_dump_start(const struct dpif *dpif OVS_UNUSED, void **iterp)
1432
{
1433
struct dp_netdev_flow_iter *iter;
1434
1435
*iterp = iter = xmalloc(sizeof *iter);
1436
iter->bucket = 0;
1437
iter->offset = 0;
1438
iter->status = 0;
1439
ovs_mutex_init(&iter->mutex);
1440
return 0;
1441
}
1442
1443
/* XXX the caller must use 'actions' without quiescing */
1444
static int
1445
dpif_netdev_flow_dump_next(const struct dpif *dpif, void *iter_, void *state_,
1446
const struct nlattr **key, size_t *key_len,
1447
const struct nlattr **mask, size_t *mask_len,
1448
const struct nlattr **actions, size_t *actions_len,
1449
const struct dpif_flow_stats **stats)
1450
{
1451
struct dp_netdev_flow_iter *iter = iter_;
1452
struct dp_netdev_flow_state *state = state_;
1453
struct dp_netdev *dp = get_dp_netdev(dpif);
1454
struct dp_netdev_flow *netdev_flow;
1455
struct flow_wildcards wc;
1456
int error;
1457
1458
ovs_mutex_lock(&iter->mutex);
1459
error = iter->status;
1460
if (!error) {
1461
struct hmap_node *node;
1462
1463
fat_rwlock_rdlock(&dp->cls.rwlock);
1464
node = hmap_at_position(&dp->flow_table, &iter->bucket, &iter->offset);
1465
if (node) {
1466
netdev_flow = CONTAINER_OF(node, struct dp_netdev_flow, node);
1467
}
1468
fat_rwlock_unlock(&dp->cls.rwlock);
1469
if (!node) {
1470
iter->status = error = EOF;
1471
}
1472
}
1473
ovs_mutex_unlock(&iter->mutex);
1474
if (error) {
1475
return error;
1476
}
1477
1478
minimask_expand(&netdev_flow->cr.match.mask, &wc);
1479
1480
if (key) {
1481
struct ofpbuf buf;
1482
1483
ofpbuf_use_stack(&buf, &state->keybuf, sizeof state->keybuf);
1484
odp_flow_key_from_flow(&buf, &netdev_flow->flow, &wc.masks,
1485
netdev_flow->flow.in_port.odp_port);
1486
1487
*key = ofpbuf_data(&buf);
1488
*key_len = ofpbuf_size(&buf);
1489
}
1490
1491
if (key && mask) {
1492
struct ofpbuf buf;
1493
1494
ofpbuf_use_stack(&buf, &state->maskbuf, sizeof state->maskbuf);
1495
odp_flow_key_from_mask(&buf, &wc.masks, &netdev_flow->flow,
1496
odp_to_u32(wc.masks.in_port.odp_port),
1497
SIZE_MAX);
1498
1499
*mask = ofpbuf_data(&buf);
1500
*mask_len = ofpbuf_size(&buf);
1501
}
1502
1503
if (actions || stats) {
1504
if (actions) {
1505
struct dp_netdev_actions *dp_actions =
1506
dp_netdev_flow_get_actions(netdev_flow);
1507
1508
*actions = dp_actions->actions;
1509
*actions_len = dp_actions->size;
1510
}
1511
1512
if (stats) {
1513
get_dpif_flow_stats(netdev_flow, &state->stats);
1514
*stats = &state->stats;
1515
}
1516
}
1517
1518
return 0;
1519
}
1520
1521
static int
1522
dpif_netdev_flow_dump_done(const struct dpif *dpif OVS_UNUSED, void *iter_)
1523
{
1524
struct dp_netdev_flow_iter *iter = iter_;
1525
1526
ovs_mutex_destroy(&iter->mutex);
1527
free(iter);
1528
return 0;
2161
static struct dpif_netdev_flow_dump *
2162
dpif_netdev_flow_dump_cast(struct dpif_flow_dump *dump)
2163
{
2164
return CONTAINER_OF(dump, struct dpif_netdev_flow_dump, up);
2165
}
2166
2167
static struct dpif_flow_dump *
2168
dpif_netdev_flow_dump_create(const struct dpif *dpif_, bool terse)
2169
{
2170
struct dpif_netdev_flow_dump *dump;
2171
2172
dump = xzalloc(sizeof *dump);
2173
dpif_flow_dump_init(&dump->up, dpif_);
2174
dump->up.terse = terse;
2175
ovs_mutex_init(&dump->mutex);
2176
2177
return &dump->up;
2178
}
2179
2180
static int
2181
dpif_netdev_flow_dump_destroy(struct dpif_flow_dump *dump_)
2182
{
2183
struct dpif_netdev_flow_dump *dump = dpif_netdev_flow_dump_cast(dump_);
2184
2185
ovs_mutex_destroy(&dump->mutex);
2186
free(dump);
2187
return 0;
2188
}
2189
2190
struct dpif_netdev_flow_dump_thread {
2191
struct dpif_flow_dump_thread up;
2192
struct dpif_netdev_flow_dump *dump;
2193
struct odputil_keybuf keybuf[FLOW_DUMP_MAX_BATCH];
2194
struct odputil_keybuf maskbuf[FLOW_DUMP_MAX_BATCH];
2195
};
2196
2197
static struct dpif_netdev_flow_dump_thread *
2198
dpif_netdev_flow_dump_thread_cast(struct dpif_flow_dump_thread *thread)
2199
{
2200
return CONTAINER_OF(thread, struct dpif_netdev_flow_dump_thread, up);
2201
}
2202
2203
static struct dpif_flow_dump_thread *
2204
dpif_netdev_flow_dump_thread_create(struct dpif_flow_dump *dump_)
2205
{
2206
struct dpif_netdev_flow_dump *dump = dpif_netdev_flow_dump_cast(dump_);
2207
struct dpif_netdev_flow_dump_thread *thread;
2208
2209
thread = xmalloc(sizeof *thread);
2210
dpif_flow_dump_thread_init(&thread->up, &dump->up);
2211
thread->dump = dump;
2212
return &thread->up;
2213
}
2214
2215
static void
2216
dpif_netdev_flow_dump_thread_destroy(struct dpif_flow_dump_thread *thread_)
2217
{
2218
struct dpif_netdev_flow_dump_thread *thread
2219
= dpif_netdev_flow_dump_thread_cast(thread_);
2220
2221
free(thread);
2222
}
2223
2224
static int
2225
dpif_netdev_flow_dump_next(struct dpif_flow_dump_thread *thread_,
2226
struct dpif_flow *flows, int max_flows)
2227
{
2228
struct dpif_netdev_flow_dump_thread *thread
2229
= dpif_netdev_flow_dump_thread_cast(thread_);
2230
struct dpif_netdev_flow_dump *dump = thread->dump;
2231
struct dp_netdev_flow *netdev_flows[FLOW_DUMP_MAX_BATCH];
2232
int n_flows = 0;
2233
int i;
2234
2235
ovs_mutex_lock(&dump->mutex);
2236
if (!dump->status) {
2237
struct dpif_netdev *dpif = dpif_netdev_cast(thread->up.dpif);
2238
struct dp_netdev *dp = get_dp_netdev(&dpif->dpif);
2239
struct dp_netdev_pmd_thread *pmd = dump->cur_pmd;
2240
int flow_limit = MIN(max_flows, FLOW_DUMP_MAX_BATCH);
2241
2242
/* First call to dump_next(), extracts the first pmd thread.
2243
* If there is no pmd thread, returns immediately. */
2244
if (!pmd) {
2245
pmd = dp_netdev_pmd_get_next(dp, &dump->poll_thread_pos);
2246
if (!pmd) {
2247
ovs_mutex_unlock(&dump->mutex);
2248
return n_flows;
2249
2250
}
2251
}
2252
2253
do {
2254
for (n_flows = 0; n_flows < flow_limit; n_flows++) {
2255
struct cmap_node *node;
2256
2257
node = cmap_next_position(&pmd->flow_table, &dump->flow_pos);
2258
if (!node) {
2259
break;
2260
}
2261
netdev_flows[n_flows] = CONTAINER_OF(node,
2262
struct dp_netdev_flow,
2263
node);
2264
}
2265
/* When finishing dumping the current pmd thread, moves to
2266
* the next. */
2267
if (n_flows < flow_limit) {
2268
memset(&dump->flow_pos, 0, sizeof dump->flow_pos);
2269
dp_netdev_pmd_unref(pmd);
2270
pmd = dp_netdev_pmd_get_next(dp, &dump->poll_thread_pos);
2271
if (!pmd) {
2272
dump->status = EOF;
2273
break;
2274
}
2275
}
2276
/* Keeps the reference to next caller. */
2277
dump->cur_pmd = pmd;
2278
2279
/* If the current dump is empty, do not exit the loop, since the
2280
* remaining pmds could have flows to be dumped. Just dumps again
2281
* on the new 'pmd'. */
2282
} while (!n_flows);
2283
}
2284
ovs_mutex_unlock(&dump->mutex);
2285
2286
for (i = 0; i < n_flows; i++) {
2287
struct odputil_keybuf *maskbuf = &thread->maskbuf[i];
2288
struct odputil_keybuf *keybuf = &thread->keybuf[i];
2289
struct dp_netdev_flow *netdev_flow = netdev_flows[i];
2290
struct dpif_flow *f = &flows[i];
2291
struct ofpbuf key, mask;
2292
2293
ofpbuf_use_stack(&key, keybuf, sizeof *keybuf);
2294
ofpbuf_use_stack(&mask, maskbuf, sizeof *maskbuf);
2295
dp_netdev_flow_to_dpif_flow(netdev_flow, &key, &mask, f,
2296
dump->up.terse);
2297
}
2298
2299
return n_flows;
1529
2300
}
1530
2301
1531
2302
static int
1532
2303
dpif_netdev_execute(struct dpif *dpif, struct dpif_execute *execute)
2304
OVS_NO_THREAD_SAFETY_ANALYSIS
1533
2305
{
1534
2306
struct dp_netdev *dp = get_dp_netdev(dpif);
1535
struct pkt_metadata *md = &execute->md;
1536
struct {
1537
struct miniflow flow;
1538
uint32_t buf[FLOW_U32S];
1539
} key;
2307
struct dp_netdev_pmd_thread *pmd;
2308
struct dp_packet *pp;
1540
2309
1541
if (ofpbuf_size(execute->packet) < ETH_HEADER_LEN ||
1542
ofpbuf_size(execute->packet) > UINT16_MAX) {
2310
if (dp_packet_size(execute->packet) < ETH_HEADER_LEN ||
2311
dp_packet_size(execute->packet) > UINT16_MAX) {
1543
2312
return EINVAL;
1544
2313
}
1545
2314
1546
/* Extract flow key. */
1547
miniflow_initialize(&key.flow, key.buf);
1548
miniflow_extract(execute->packet, md, &key.flow);
1549
1550
ovs_rwlock_rdlock(&dp->port_rwlock);
1551
dp_netdev_execute_actions(dp, &key.flow, execute->packet, false, md,
1552
execute->actions, execute->actions_len);
1553
ovs_rwlock_unlock(&dp->port_rwlock);
2315
/* Tries finding the 'pmd'. If NULL is returned, that means
2316
* the current thread is a non-pmd thread and should use
2317
* dp_netdev_get_pmd(dp, NON_PMD_CORE_ID). */
2318
pmd = ovsthread_getspecific(dp->per_pmd_key);
2319
if (!pmd) {
2320
pmd = dp_netdev_get_pmd(dp, NON_PMD_CORE_ID);
2321
}
2322
2323
/* If the current thread is non-pmd thread, acquires
2324
* the 'non_pmd_mutex'. */
2325
if (pmd->core_id == NON_PMD_CORE_ID) {
2326
ovs_mutex_lock(&dp->non_pmd_mutex);
2327
ovs_mutex_lock(&dp->port_mutex);
2328
}
2329
2330
pp = execute->packet;
2331
dp_netdev_execute_actions(pmd, &pp, 1, false, execute->actions,
2332
execute->actions_len);
2333
if (pmd->core_id == NON_PMD_CORE_ID) {
2334
dp_netdev_pmd_unref(pmd);
2335
ovs_mutex_unlock(&dp->port_mutex);
2336
ovs_mutex_unlock(&dp->non_pmd_mutex);
2337
}
1554
2338
1555
2339
return 0;
1556
2340
}
1557
2341
1558
2342
static void
1559
dp_netdev_destroy_all_queues(struct dp_netdev *dp)
1560
OVS_REQ_WRLOCK(dp->queue_rwlock)
2343
dpif_netdev_operate(struct dpif *dpif, struct dpif_op **ops, size_t n_ops)
1561
2344
{
1562
2345
size_t i;
1563
2346
1564
dp_netdev_purge_queues(dp);
1565
1566
for (i = 0; i < dp->n_handlers; i++) {
1567
struct dp_netdev_queue *q = &dp->handler_queues[i];
1568
1569
ovs_mutex_destroy(&q->mutex);
1570
seq_destroy(q->seq);
1571
}
1572
free(dp->handler_queues);
1573
dp->handler_queues = NULL;
1574
dp->n_handlers = 0;
1575
}
1576
1577
static void
1578
dp_netdev_refresh_queues(struct dp_netdev *dp, uint32_t n_handlers)
1579
OVS_REQ_WRLOCK(dp->queue_rwlock)
1580
{
1581
if (dp->n_handlers != n_handlers) {
1582
size_t i;
1583
1584
dp_netdev_destroy_all_queues(dp);
1585
1586
dp->n_handlers = n_handlers;
1587
dp->handler_queues = xzalloc(n_handlers * sizeof *dp->handler_queues);
1588
1589
for (i = 0; i < n_handlers; i++) {
1590
struct dp_netdev_queue *q = &dp->handler_queues[i];
1591
1592
ovs_mutex_init(&q->mutex);
1593
q->seq = seq_create();
2347
for (i = 0; i < n_ops; i++) {
2348
struct dpif_op *op = ops[i];
2349
2350
switch (op->type) {
2351
case DPIF_OP_FLOW_PUT:
2352
op->error = dpif_netdev_flow_put(dpif, &op->u.flow_put);
2353
break;
2354
2355
case DPIF_OP_FLOW_DEL:
2356
op->error = dpif_netdev_flow_del(dpif, &op->u.flow_del);
2357
break;
2358
2359
case DPIF_OP_EXECUTE:
2360
op->error = dpif_netdev_execute(dpif, &op->u.execute);
2361
break;
2362
2363
case DPIF_OP_FLOW_GET:
2364
op->error = dpif_netdev_flow_get(dpif, &op->u.flow_get);
2365
break;
1594
2366
}
1595
2367
}
1596
2368
}
1597
2369
1598
static int
1599
dpif_netdev_recv_set(struct dpif *dpif, bool enable)
2370
/* Returns true if the configuration for rx queues or cpu mask
2371
* is changed. */
2372
static bool
2373
pmd_config_changed(const struct dp_netdev *dp, size_t rxqs, const char *cmask)
1600
2374
{
1601
struct dp_netdev *dp = get_dp_netdev(dpif);
1602
1603
if ((dp->handler_queues != NULL) == enable) {
1604
return 0;
1605
}
1606
1607
fat_rwlock_wrlock(&dp->queue_rwlock);
1608
if (!enable) {
1609
dp_netdev_destroy_all_queues(dp);
2375
if (dp->n_dpdk_rxqs != rxqs) {
2376
return true;
1610
2377
} else {
1611
dp_netdev_refresh_queues(dp, 1);
2378
if (dp->pmd_cmask != NULL && cmask != NULL) {
2379
return strcmp(dp->pmd_cmask, cmask);
2380
} else {
2381
return (dp->pmd_cmask != NULL || cmask != NULL);
2382
}
1612
2383
}
1613
fat_rwlock_unlock(&dp->queue_rwlock);
1614
1615
return 0;
1616
2384
}
1617
2385
2386
/* Resets pmd threads if the configuration for 'rxq's or cpu mask changes. */
1618
2387
static int
1619
dpif_netdev_handlers_set(struct dpif *dpif, uint32_t n_handlers)
2388
dpif_netdev_pmd_set(struct dpif *dpif, unsigned int n_rxqs, const char *cmask)
1620
2389
{
1621
2390
struct dp_netdev *dp = get_dp_netdev(dpif);
1622
2391
1623
fat_rwlock_wrlock(&dp->queue_rwlock);
1624
if (dp->handler_queues) {
1625
dp_netdev_refresh_queues(dp, n_handlers);
2392
if (pmd_config_changed(dp, n_rxqs, cmask)) {
2393
struct dp_netdev_port *port;
2394
2395
dp_netdev_destroy_all_pmds(dp);
2396
2397
CMAP_FOR_EACH (port, node, &dp->ports) {
2398
if (netdev_is_pmd(port->netdev)) {
2399
int i, err;
2400
2401
/* Closes the existing 'rxq's. */
2402
for (i = 0; i < netdev_n_rxq(port->netdev); i++) {
2403
netdev_rxq_close(port->rxq[i]);
2404
port->rxq[i] = NULL;
2405
}
2406
2407
/* Sets the new rx queue config. */
2408
err = netdev_set_multiq(port->netdev,
2409
ovs_numa_get_n_cores() + 1,
2410
n_rxqs);
2411
if (err && (err != EOPNOTSUPP)) {
2412
VLOG_ERR("Failed to set dpdk interface %s rx_queue to:"
2413
" %u", netdev_get_name(port->netdev),
2414
n_rxqs);
2415
return err;
2416
}
2417
2418
/* If the set_multiq() above succeeds, reopens the 'rxq's. */
2419
port->rxq = xrealloc(port->rxq, sizeof *port->rxq
2420
* netdev_n_rxq(port->netdev));
2421
for (i = 0; i < netdev_n_rxq(port->netdev); i++) {
2422
netdev_rxq_open(port->netdev, &port->rxq[i], i);
2423
}
2424
}
2425
}
2426
dp->n_dpdk_rxqs = n_rxqs;
2427
2428
/* Reconfigures the cpu mask. */
2429
ovs_numa_set_cpu_mask(cmask);
2430
free(dp->pmd_cmask);
2431
dp->pmd_cmask = cmask ? xstrdup(cmask) : NULL;
2432
2433
/* Restores the non-pmd. */
2434
dp_netdev_set_nonpmd(dp);
2435
/* Restores all pmd threads. */
2436
dp_netdev_reset_pmd_threads(dp);
1626
2437
}
1627
fat_rwlock_unlock(&dp->queue_rwlock);
1628
2438
1629
2439
return 0;
1630
2440
}
1637
2447
return 0;
1638
2448
}
1639
2449
1640
static bool
1641
dp_netdev_recv_check(const struct dp_netdev *dp, const uint32_t handler_id)
1642
OVS_REQ_RDLOCK(dp->queue_rwlock)
1643
{
1644
static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
1645
1646
if (!dp->handler_queues) {
1647
VLOG_WARN_RL(&rl, "receiving upcall disabled");
1648
return false;
1649
}
1650
1651
if (handler_id >= dp->n_handlers) {
1652
VLOG_WARN_RL(&rl, "handler index out of bound");
1653
return false;
1654
}
1655
1656
return true;
1657
}
1658
1659
static int
1660
dpif_netdev_recv(struct dpif *dpif, uint32_t handler_id,
1661
struct dpif_upcall *upcall, struct ofpbuf *buf)
1662
{
1663
struct dp_netdev *dp = get_dp_netdev(dpif);
1664
struct dp_netdev_queue *q;
1665
int error = 0;
1666
1667
fat_rwlock_rdlock(&dp->queue_rwlock);
1668
1669
if (!dp_netdev_recv_check(dp, handler_id)) {
1670
error = EAGAIN;
1671
goto out;
1672
}
1673
1674
q = &dp->handler_queues[handler_id];
1675
ovs_mutex_lock(&q->mutex);
1676
if (q->head != q->tail) {
1677
struct dp_netdev_upcall *u = &q->upcalls[q->tail++ & QUEUE_MASK];
1678
1679
*upcall = u->upcall;
1680
1681
ofpbuf_uninit(buf);
1682
*buf = u->buf;
1683
} else {
1684
error = EAGAIN;
1685
}
1686
ovs_mutex_unlock(&q->mutex);
1687
1688
out:
1689
fat_rwlock_unlock(&dp->queue_rwlock);
1690
1691
return error;
1692
}
1693
1694
static void
1695
dpif_netdev_recv_wait(struct dpif *dpif, uint32_t handler_id)
1696
{
1697
struct dp_netdev *dp = get_dp_netdev(dpif);
1698
struct dp_netdev_queue *q;
1699
uint64_t seq;
1700
1701
fat_rwlock_rdlock(&dp->queue_rwlock);
1702
1703
if (!dp_netdev_recv_check(dp, handler_id)) {
1704
goto out;
1705
}
1706
1707
q = &dp->handler_queues[handler_id];
1708
ovs_mutex_lock(&q->mutex);
1709
seq = seq_read(q->seq);
1710
if (q->head != q->tail) {
1711
poll_immediate_wake();
1712
} else {
1713
seq_wait(q->seq, seq);
1714
}
1715
1716
ovs_mutex_unlock(&q->mutex);
1717
1718
out:
1719
fat_rwlock_unlock(&dp->queue_rwlock);
1720
}
1721
1722
static void
1723
dpif_netdev_recv_purge(struct dpif *dpif)
1724
{
1725
struct dpif_netdev *dpif_netdev = dpif_netdev_cast(dpif);
1726
1727
fat_rwlock_wrlock(&dpif_netdev->dp->queue_rwlock);
1728
dp_netdev_purge_queues(dpif_netdev->dp);
1729
fat_rwlock_unlock(&dpif_netdev->dp->queue_rwlock);
1730
}
1731
2450
1732
/* Creates and returns a new 'struct dp_netdev_actions', with a reference count
1733
* of 1, whose actions are a copy of from the 'ofpacts_len' bytes of
1734
* 'ofpacts'. */
2451
/* Creates and returns a new 'struct dp_netdev_actions', whose actions are
2452
* a copy of the 'ofpacts_len' bytes of 'ofpacts'. */
1735
2453
struct dp_netdev_actions *
1736
2454
dp_netdev_actions_create(const struct nlattr *actions, size_t size)
1737
2455
{
1738
2456
struct dp_netdev_actions *netdev_actions;
1739
2457
1740
netdev_actions = xmalloc(sizeof *netdev_actions);
1741
netdev_actions->actions = xmemdup(actions, size);
2458
netdev_actions = xmalloc(sizeof *netdev_actions + size);
2459
memcpy(netdev_actions->actions, actions, size);
1742
2460
netdev_actions->size = size;
1743
2461
1744
2462
return netdev_actions;
1753
2471
static void
1754
2472
dp_netdev_actions_free(struct dp_netdev_actions *actions)
1755
2473
{
1756
free(actions->actions);
1757
2474
free(actions);
1758
2475
}
1759
2476
2477
static inline unsigned long long
2478
cycles_counter(void)
2479
{
2480
#ifdef DPDK_NETDEV
2481
return rte_get_tsc_cycles();
2482
#else
2483
return 0;
2484
#endif
2485
}
2486
2487
/* Fake mutex to make sure that the calls to cycles_count_* are balanced */
2488
extern struct ovs_mutex cycles_counter_fake_mutex;
2489
2490
/* Start counting cycles. Must be followed by 'cycles_count_end()' */
2491
static inline void
2492
cycles_count_start(struct dp_netdev_pmd_thread *pmd)
2493
OVS_ACQUIRES(&cycles_counter_fake_mutex)
2494
OVS_NO_THREAD_SAFETY_ANALYSIS
2495
{
2496
pmd->last_cycles = cycles_counter();
2497
}
2498
2499
/* Stop counting cycles and add them to the counter 'type' */
2500
static inline void
2501
cycles_count_end(struct dp_netdev_pmd_thread *pmd,
2502
enum pmd_cycles_counter_type type)
2503
OVS_RELEASES(&cycles_counter_fake_mutex)
2504
OVS_NO_THREAD_SAFETY_ANALYSIS
2505
{
2506
unsigned long long interval = cycles_counter() - pmd->last_cycles;
2507
2508
non_atomic_ullong_add(&pmd->cycles.n[type], interval);
2509
}
1760
2510
1761
2511
static void
1762
dp_netdev_process_rxq_port(struct dp_netdev *dp,
1763
struct dp_netdev_port *port,
1764
struct netdev_rxq *rxq)
2512
dp_netdev_process_rxq_port(struct dp_netdev_pmd_thread *pmd,
2513
struct dp_netdev_port *port,
2514
struct netdev_rxq *rxq)
1765
2515
{
1766
struct ofpbuf *packet[NETDEV_MAX_RX_BATCH];
1767
int error, c;
2516
struct dp_packet *packets[NETDEV_MAX_BURST];
2517
int error, cnt;
1768
2518
1769
error = netdev_rxq_recv(rxq, packet, &c);
2519
cycles_count_start(pmd);
2520
error = netdev_rxq_recv(rxq, packets, &cnt);
2521
cycles_count_end(pmd, PMD_CYCLES_POLLING);
1770
2522
if (!error) {
1771
struct pkt_metadata md = PKT_METADATA_INITIALIZER(port->port_no);
1772
2523
int i;
1773
2524
1774
for (i = 0; i < c; i++) {
1775
dp_netdev_port_input(dp, packet[i], &md);
2525
*recirc_depth_get() = 0;
2526
2527
/* XXX: initialize md in netdev implementation. */
2528
for (i = 0; i < cnt; i++) {
2529
packets[i]->md = port->md;
1776
2530
}
2531
cycles_count_start(pmd);
2532
dp_netdev_input(pmd, packets, cnt);
2533
cycles_count_end(pmd, PMD_CYCLES_PROCESSING);
1777
2534
} else if (error != EAGAIN && error != EOPNOTSUPP) {
1778
static struct vlog_rate_limit rl
1779
= VLOG_RATE_LIMIT_INIT(1, 5);
2535
static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
1780
2536
1781
2537
VLOG_ERR_RL(&rl, "error receiving data from %s: %s",
1782
netdev_get_name(port->netdev),
1783
ovs_strerror(error));
2538
netdev_get_name(port->netdev), ovs_strerror(error));
1784
2539
}
1785
2540
}
1786
2541
1787
static void
2542
/* Return true if needs to revalidate datapath flows. */
2543
static bool
1788
2544
dpif_netdev_run(struct dpif *dpif)
1789
2545
{
1790
2546
struct dp_netdev_port *port;
1791
2547
struct dp_netdev *dp = get_dp_netdev(dpif);
1792
1793
ovs_rwlock_rdlock(&dp->port_rwlock);
1794
1795
HMAP_FOR_EACH (port, node, &dp->ports) {
2548
struct dp_netdev_pmd_thread *non_pmd = dp_netdev_get_pmd(dp,
2549
NON_PMD_CORE_ID);
2550
uint64_t new_tnl_seq;
2551
2552
ovs_mutex_lock(&dp->non_pmd_mutex);
2553
CMAP_FOR_EACH (port, node, &dp->ports) {
1796
2554
if (!netdev_is_pmd(port->netdev)) {
1797
2555
int i;
1798
2556
1799
2557
for (i = 0; i < netdev_n_rxq(port->netdev); i++) {
1800
dp_netdev_process_rxq_port(dp, port, port->rxq[i]);
2558
dp_netdev_process_rxq_port(non_pmd, port, port->rxq[i]);
1801
2559
}
1802
2560
}
1803
2561
}
1804
1805
ovs_rwlock_unlock(&dp->port_rwlock);
2562
ovs_mutex_unlock(&dp->non_pmd_mutex);
2563
dp_netdev_pmd_unref(non_pmd);
2564
2565
tnl_arp_cache_run();
2566
new_tnl_seq = seq_read(tnl_conf_seq);
2567
2568
if (dp->last_tnl_conf_seq != new_tnl_seq) {
2569
dp->last_tnl_conf_seq = new_tnl_seq;
2570
return true;
2571
}
2572
return false;
1806
2573
}
1807
2574
1808
2575
static void
1811
2578
struct dp_netdev_port *port;
1812
2579
struct dp_netdev *dp = get_dp_netdev(dpif);
1813
2580
1814
ovs_rwlock_rdlock(&dp->port_rwlock);
1815
1816
HMAP_FOR_EACH (port, node, &dp->ports) {
2581
ovs_mutex_lock(&dp_netdev_mutex);
2582
CMAP_FOR_EACH (port, node, &dp->ports) {
1817
2583
if (!netdev_is_pmd(port->netdev)) {
1818
2584
int i;
1819
2585
1822
2588
}
1823
2589
}
1824
2590
}
1825
ovs_rwlock_unlock(&dp->port_rwlock);
2591
ovs_mutex_unlock(&dp_netdev_mutex);
2592
seq_wait(tnl_conf_seq, dp->last_tnl_conf_seq);
1826
2593
}
1827
2594
1828
2595
struct rxq_poll {
1831
2598
};
1832
2599
1833
2600
static int
1834
pmd_load_queues(struct pmd_thread *f,
2601
pmd_load_queues(struct dp_netdev_pmd_thread *pmd,
1835
2602
struct rxq_poll **ppoll_list, int poll_cnt)
1836
2603
{
1837
struct dp_netdev *dp = f->dp;
1838
2604
struct rxq_poll *poll_list = *ppoll_list;
1839
2605
struct dp_netdev_port *port;
1840
int id = f->id;
1841
int index;
1842
int i;
2606
int n_pmds_on_numa, index, i;
1843
2607
1844
2608
/* Simple scheduler for netdev rx polling. */
1845
ovs_rwlock_rdlock(&dp->port_rwlock);
1846
2609
for (i = 0; i < poll_cnt; i++) {
1847
port_unref(poll_list[i].port);
2610
port_unref(poll_list[i].port);
1848
2611
}
1849
2612
1850
2613
poll_cnt = 0;
2614
n_pmds_on_numa = get_n_pmd_threads_on_numa(pmd->dp, pmd->numa_id);
1851
2615
index = 0;
1852
2616
1853
HMAP_FOR_EACH (port, node, &f->dp->ports) {
1854
if (netdev_is_pmd(port->netdev)) {
1855
int i;
1856
1857
for (i = 0; i < netdev_n_rxq(port->netdev); i++) {
1858
if ((index % dp->n_pmd_threads) == id) {
1859
poll_list = xrealloc(poll_list, sizeof *poll_list * (poll_cnt + 1));
1860
1861
port_ref(port);
1862
poll_list[poll_cnt].port = port;
1863
poll_list[poll_cnt].rx = port->rxq[i];
1864
poll_cnt++;
2617
CMAP_FOR_EACH (port, node, &pmd->dp->ports) {
2618
/* Calls port_try_ref() to prevent the main thread
2619
* from deleting the port. */
2620
if (port_try_ref(port)) {
2621
if (netdev_is_pmd(port->netdev)
2622
&& netdev_get_numa_id(port->netdev) == pmd->numa_id) {
2623
int i;
2624
2625
for (i = 0; i < netdev_n_rxq(port->netdev); i++) {
2626
if ((index % n_pmds_on_numa) == pmd->index) {
2627
poll_list = xrealloc(poll_list,
2628
sizeof *poll_list * (poll_cnt + 1));
2629
2630
port_ref(port);
2631
poll_list[poll_cnt].port = port;
2632
poll_list[poll_cnt].rx = port->rxq[i];
2633
poll_cnt++;
2634
}
2635
index++;
1865
2636
}
1866
index++;
1867
2637
}
2638
/* Unrefs the port_try_ref(). */
2639
port_unref(port);
1868
2640
}
1869
2641
}
1870
2642
1871
ovs_rwlock_unlock(&dp->port_rwlock);
1872
2643
*ppoll_list = poll_list;
1873
2644
return poll_cnt;
1874
2645
}
1876
2647
static void *
1877
2648
pmd_thread_main(void *f_)
1878
2649
{
1879
struct pmd_thread *f = f_;
1880
struct dp_netdev *dp = f->dp;
2650
struct dp_netdev_pmd_thread *pmd = f_;
1881
2651
unsigned int lc = 0;
1882
2652
struct rxq_poll *poll_list;
1883
unsigned int port_seq;
2653
unsigned int port_seq = PMD_INITIAL_SEQ;
1884
2654
int poll_cnt;
1885
2655
int i;
1886
2656
1887
2657
poll_cnt = 0;
1888
2658
poll_list = NULL;
1889
2659
1890
pmd_thread_setaffinity_cpu(f->id);
2660
/* Stores the pmd thread's 'pmd' to 'per_pmd_key'. */
2661
ovsthread_setspecific(pmd->dp->per_pmd_key, pmd);
2662
pmd_thread_setaffinity_cpu(pmd->core_id);
1891
2663
reload:
1892
poll_cnt = pmd_load_queues(f, &poll_list, poll_cnt);
1893
atomic_read(&f->change_seq, &port_seq);
2664
emc_cache_init(&pmd->flow_cache);
2665
poll_cnt = pmd_load_queues(pmd, &poll_list, poll_cnt);
2666
2667
/* List port/core affinity */
2668
for (i = 0; i < poll_cnt; i++) {
2669
VLOG_INFO("Core %d processing port \'%s\'\n", pmd->core_id, netdev_get_name(poll_list[i].port->netdev));
2670
}
2671
2672
/* Signal here to make sure the pmd finishes
2673
* reloading the updated configuration. */
2674
dp_netdev_pmd_reload_done(pmd);
1894
2675
1895
2676
for (;;) {
1896
unsigned int c_port_seq;
1897
2677
int i;
1898
2678
1899
2679
for (i = 0; i < poll_cnt; i++) {
1900
dp_netdev_process_rxq_port(dp, poll_list[i].port, poll_list[i].rx);
2680
dp_netdev_process_rxq_port(pmd, poll_list[i].port, poll_list[i].rx);
1901
2681
}
1902
2682
1903
2683
if (lc++ > 1024) {
2684
unsigned int seq;
2685
2686
lc = 0;
2687
2688
emc_cache_slow_sweep(&pmd->flow_cache);
1904
2689
ovsrcu_quiesce();
1905
2690
1906
/* TODO: need completely userspace based signaling method.
1907
* to keep this thread entirely in userspace.
1908
* For now using atomic counter. */
1909
lc = 0;
1910
atomic_read_explicit(&f->change_seq, &c_port_seq, memory_order_consume);
1911
if (c_port_seq != port_seq) {
2691
atomic_read_relaxed(&pmd->change_seq, &seq);
2692
if (seq != port_seq) {
2693
port_seq = seq;
1912
2694
break;
1913
2695
}
1914
2696
}
1915
2697
}
1916
2698
1917
if (!latch_is_set(&f->dp->exit_latch)){
2699
emc_cache_uninit(&pmd->flow_cache);
2700
2701
if (!latch_is_set(&pmd->exit_latch)){
1918
2702
goto reload;
1919
2703
}
1920
2704
1922
2706
port_unref(poll_list[i].port);
1923
2707
}
1924
2708
2709
dp_netdev_pmd_reload_done(pmd);
2710
1925
2711
free(poll_list);
1926
2712
return NULL;
1927
2713
}
1928
2714
1929
2715
static void
1930
dp_netdev_set_pmd_threads(struct dp_netdev *dp, int n)
1931
{
1932
int i;
1933
1934
if (n == dp->n_pmd_threads) {
1935
return;
1936
}
1937
1938
/* Stop existing threads. */
1939
latch_set(&dp->exit_latch);
1940
dp_netdev_reload_pmd_threads(dp);
1941
for (i = 0; i < dp->n_pmd_threads; i++) {
1942
struct pmd_thread *f = &dp->pmd_threads[i];
1943
1944
xpthread_join(f->thread, NULL);
1945
}
1946
latch_poll(&dp->exit_latch);
1947
free(dp->pmd_threads);
1948
1949
/* Start new threads. */
1950
dp->pmd_threads = xmalloc(n * sizeof *dp->pmd_threads);
1951
dp->n_pmd_threads = n;
1952
1953
for (i = 0; i < n; i++) {
1954
struct pmd_thread *f = &dp->pmd_threads[i];
1955
1956
f->dp = dp;
1957
f->id = i;
1958
atomic_store(&f->change_seq, 1);
1959
1960
/* Each thread will distribute all devices rx-queues among
1961
* themselves. */
1962
f->thread = ovs_thread_create("pmd", pmd_thread_main, f);
2716
dp_netdev_disable_upcall(struct dp_netdev *dp)
2717
OVS_ACQUIRES(dp->upcall_rwlock)
2718
{
2719
fat_rwlock_wrlock(&dp->upcall_rwlock);
2720
}
2721
2722
static void
2723
dpif_netdev_disable_upcall(struct dpif *dpif)
2724
OVS_NO_THREAD_SAFETY_ANALYSIS
2725
{
2726
struct dp_netdev *dp = get_dp_netdev(dpif);
2727
dp_netdev_disable_upcall(dp);
2728
}
2729
2730
static void
2731
dp_netdev_enable_upcall(struct dp_netdev *dp)
2732
OVS_RELEASES(dp->upcall_rwlock)
2733
{
2734
fat_rwlock_unlock(&dp->upcall_rwlock);
2735
}
2736
2737
static void
2738
dpif_netdev_enable_upcall(struct dpif *dpif)
2739
OVS_NO_THREAD_SAFETY_ANALYSIS
2740
{
2741
struct dp_netdev *dp = get_dp_netdev(dpif);
2742
dp_netdev_enable_upcall(dp);
2743
}
2744
2745
void
2746
dp_netdev_pmd_reload_done(struct dp_netdev_pmd_thread *pmd)
2747
{
2748
ovs_mutex_lock(&pmd->cond_mutex);
2749
xpthread_cond_signal(&pmd->cond);
2750
ovs_mutex_unlock(&pmd->cond_mutex);
2751
}
2752
2753
/* Finds and refs the dp_netdev_pmd_thread on core 'core_id'. Returns
2754
* the pointer if succeeds, otherwise, NULL.
2755
*
2756
* Caller must unrefs the returned reference. */
2757
static struct dp_netdev_pmd_thread *
2758
dp_netdev_get_pmd(struct dp_netdev *dp, unsigned core_id)
2759
{
2760
struct dp_netdev_pmd_thread *pmd;
2761
const struct cmap_node *pnode;
2762
2763
pnode = cmap_find(&dp->poll_threads, hash_int(core_id, 0));
2764
if (!pnode) {
2765
return NULL;
2766
}
2767
pmd = CONTAINER_OF(pnode, struct dp_netdev_pmd_thread, node);
2768
2769
return dp_netdev_pmd_try_ref(pmd) ? pmd : NULL;
2770
}
2771
2772
/* Sets the 'struct dp_netdev_pmd_thread' for non-pmd threads. */
2773
static void
2774
dp_netdev_set_nonpmd(struct dp_netdev *dp)
2775
{
2776
struct dp_netdev_pmd_thread *non_pmd;
2777
2778
non_pmd = xzalloc(sizeof *non_pmd);
2779
dp_netdev_configure_pmd(non_pmd, dp, 0, NON_PMD_CORE_ID,
2780
OVS_NUMA_UNSPEC);
2781
}
2782
2783
/* Caller must have valid pointer to 'pmd'. */
2784
static bool
2785
dp_netdev_pmd_try_ref(struct dp_netdev_pmd_thread *pmd)
2786
{
2787
return ovs_refcount_try_ref_rcu(&pmd->ref_cnt);
2788
}
2789
2790
static void
2791
dp_netdev_pmd_unref(struct dp_netdev_pmd_thread *pmd)
2792
{
2793
if (pmd && ovs_refcount_unref(&pmd->ref_cnt) == 1) {
2794
ovsrcu_postpone(dp_netdev_destroy_pmd, pmd);
2795
}
2796
}
2797
2798
/* Given cmap position 'pos', tries to ref the next node. If try_ref()
2799
* fails, keeps checking for next node until reaching the end of cmap.
2800
*
2801
* Caller must unrefs the returned reference. */
2802
static struct dp_netdev_pmd_thread *
2803
dp_netdev_pmd_get_next(struct dp_netdev *dp, struct cmap_position *pos)
2804
{
2805
struct dp_netdev_pmd_thread *next;
2806
2807
do {
2808
struct cmap_node *node;
2809
2810
node = cmap_next_position(&dp->poll_threads, pos);
2811
next = node ? CONTAINER_OF(node, struct dp_netdev_pmd_thread, node)
2812
: NULL;
2813
} while (next && !dp_netdev_pmd_try_ref(next));
2814
2815
return next;
2816
}
2817
2818
static int
2819
core_id_to_qid(unsigned core_id)
2820
{
2821
if (core_id != NON_PMD_CORE_ID) {
2822
return core_id;
2823
} else {
2824
return ovs_numa_get_n_cores();
2825
}
2826
}
2827
2828
/* Configures the 'pmd' based on the input argument. */
2829
static void
2830
dp_netdev_configure_pmd(struct dp_netdev_pmd_thread *pmd, struct dp_netdev *dp,
2831
int index, unsigned core_id, int numa_id)
2832
{
2833
pmd->dp = dp;
2834
pmd->index = index;
2835
pmd->core_id = core_id;
2836
pmd->tx_qid = core_id_to_qid(core_id);
2837
pmd->numa_id = numa_id;
2838
2839
ovs_refcount_init(&pmd->ref_cnt);
2840
latch_init(&pmd->exit_latch);
2841
atomic_init(&pmd->change_seq, PMD_INITIAL_SEQ);
2842
xpthread_cond_init(&pmd->cond, NULL);
2843
ovs_mutex_init(&pmd->cond_mutex);
2844
ovs_mutex_init(&pmd->flow_mutex);
2845
dpcls_init(&pmd->cls);
2846
cmap_init(&pmd->flow_table);
2847
/* init the 'flow_cache' since there is no
2848
* actual thread created for NON_PMD_CORE_ID. */
2849
if (core_id == NON_PMD_CORE_ID) {
2850
emc_cache_init(&pmd->flow_cache);
2851
}
2852
cmap_insert(&dp->poll_threads, CONST_CAST(struct cmap_node *, &pmd->node),
2853
hash_int(core_id, 0));
2854
}
2855
2856
static void
2857
dp_netdev_destroy_pmd(struct dp_netdev_pmd_thread *pmd)
2858
{
2859
dp_netdev_pmd_flow_flush(pmd);
2860
dpcls_destroy(&pmd->cls);
2861
cmap_destroy(&pmd->flow_table);
2862
ovs_mutex_destroy(&pmd->flow_mutex);
2863
latch_destroy(&pmd->exit_latch);
2864
xpthread_cond_destroy(&pmd->cond);
2865
ovs_mutex_destroy(&pmd->cond_mutex);
2866
free(pmd);
2867
}
2868
2869
/* Stops the pmd thread, removes it from the 'dp->poll_threads',
2870
* and unrefs the struct. */
2871
static void
2872
dp_netdev_del_pmd(struct dp_netdev_pmd_thread *pmd)
2873
{
2874
/* Uninit the 'flow_cache' since there is
2875
* no actual thread uninit it for NON_PMD_CORE_ID. */
2876
if (pmd->core_id == NON_PMD_CORE_ID) {
2877
emc_cache_uninit(&pmd->flow_cache);
2878
} else {
2879
latch_set(&pmd->exit_latch);
2880
dp_netdev_reload_pmd__(pmd);
2881
ovs_numa_unpin_core(pmd->core_id);
2882
xpthread_join(pmd->thread, NULL);
2883
}
2884
cmap_remove(&pmd->dp->poll_threads, &pmd->node, hash_int(pmd->core_id, 0));
2885
dp_netdev_pmd_unref(pmd);
2886
}
2887
2888
/* Destroys all pmd threads. */
2889
static void
2890
dp_netdev_destroy_all_pmds(struct dp_netdev *dp)
2891
{
2892
struct dp_netdev_pmd_thread *pmd;
2893
2894
CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
2895
dp_netdev_del_pmd(pmd);
2896
}
2897
}
2898
2899
/* Deletes all pmd threads on numa node 'numa_id'. */
2900
static void
2901
dp_netdev_del_pmds_on_numa(struct dp_netdev *dp, int numa_id)
2902
{
2903
struct dp_netdev_pmd_thread *pmd;
2904
2905
CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
2906
if (pmd->numa_id == numa_id) {
2907
dp_netdev_del_pmd(pmd);
2908
}
2909
}
2910
}
2911
2912
/* Checks the numa node id of 'netdev' and starts pmd threads for
2913
* the numa node. */
2914
static void
2915
dp_netdev_set_pmds_on_numa(struct dp_netdev *dp, int numa_id)
2916
{
2917
int n_pmds;
2918
2919
if (!ovs_numa_numa_id_is_valid(numa_id)) {
2920
VLOG_ERR("Cannot create pmd threads due to numa id (%d)"
2921
"invalid", numa_id);
2922
return ;
2923
}
2924
2925
n_pmds = get_n_pmd_threads_on_numa(dp, numa_id);
2926
2927
/* If there are already pmd threads created for the numa node
2928
* in which 'netdev' is on, do nothing. Else, creates the
2929
* pmd threads for the numa node. */
2930
if (!n_pmds) {
2931
int can_have, n_unpinned, i;
2932
2933
n_unpinned = ovs_numa_get_n_unpinned_cores_on_numa(numa_id);
2934
if (!n_unpinned) {
2935
VLOG_ERR("Cannot create pmd threads due to out of unpinned "
2936
"cores on numa node");
2937
return;
2938
}
2939
2940
/* If cpu mask is specified, uses all unpinned cores, otherwise
2941
* tries creating NR_PMD_THREADS pmd threads. */
2942
can_have = dp->pmd_cmask ? n_unpinned : MIN(n_unpinned, NR_PMD_THREADS);
2943
for (i = 0; i < can_have; i++) {
2944
struct dp_netdev_pmd_thread *pmd = xzalloc(sizeof *pmd);
2945
unsigned core_id = ovs_numa_get_unpinned_core_on_numa(numa_id);
2946
2947
dp_netdev_configure_pmd(pmd, dp, i, core_id, numa_id);
2948
/* Each thread will distribute all devices rx-queues among
2949
* themselves. */
2950
pmd->thread = ovs_thread_create("pmd", pmd_thread_main, pmd);
2951
}
2952
VLOG_INFO("Created %d pmd threads on numa node %d", can_have, numa_id);
1963
2953
}
1964
2954
}
1965
2955
1966
2956
1967
static void *
1968
dp_netdev_flow_stats_new_cb(void)
1969
{
1970
struct dp_netdev_flow_stats *bucket = xzalloc_cacheline(sizeof *bucket);
1971
ovs_mutex_init(&bucket->mutex);
1972
return bucket;
1973
}
1974
1975
static void
1976
dp_netdev_flow_used(struct dp_netdev_flow *netdev_flow,
1977
const struct ofpbuf *packet,
1978
const struct miniflow *key)
1979
{
1980
uint16_t tcp_flags = miniflow_get_tcp_flags(key);
1981
long long int now = time_msec();
1982
struct dp_netdev_flow_stats *bucket;
1983
1984
bucket = ovsthread_stats_bucket_get(&netdev_flow->stats,
1985
dp_netdev_flow_stats_new_cb);
1986
1987
ovs_mutex_lock(&bucket->mutex);
1988
bucket->used = MAX(now, bucket->used);
1989
bucket->packet_count++;
1990
bucket->byte_count += ofpbuf_size(packet);
1991
bucket->tcp_flags |= tcp_flags;
1992
ovs_mutex_unlock(&bucket->mutex);
1993
}
1994
1995
static void *
1996
dp_netdev_stats_new_cb(void)
1997
{
1998
struct dp_netdev_stats *bucket = xzalloc_cacheline(sizeof *bucket);
1999
ovs_mutex_init(&bucket->mutex);
2000
return bucket;
2001
}
2002
2003
static void
2004
dp_netdev_count_packet(struct dp_netdev *dp, enum dp_stat_type type)
2005
{
2006
struct dp_netdev_stats *bucket;
2007
2008
bucket = ovsthread_stats_bucket_get(&dp->stats, dp_netdev_stats_new_cb);
2009
ovs_mutex_lock(&bucket->mutex);
2010
bucket->n[type]++;
2011
ovs_mutex_unlock(&bucket->mutex);
2012
}
2013
2014
static void
2015
dp_netdev_input(struct dp_netdev *dp, struct ofpbuf *packet,
2016
struct pkt_metadata *md)
2017
OVS_REQ_RDLOCK(dp->port_rwlock)
2018
{
2019
struct dp_netdev_flow *netdev_flow;
2020
struct {
2021
struct miniflow flow;
2022
uint32_t buf[FLOW_U32S];
2023
} key;
2024
2025
if (ofpbuf_size(packet) < ETH_HEADER_LEN) {
2026
ofpbuf_delete(packet);
2957
/* Called after pmd threads config change. Restarts pmd threads with
2958
* new configuration. */
2959
static void
2960
dp_netdev_reset_pmd_threads(struct dp_netdev *dp)
2961
{
2962
struct dp_netdev_port *port;
2963
2964
CMAP_FOR_EACH (port, node, &dp->ports) {
2965
if (netdev_is_pmd(port->netdev)) {
2966
int numa_id = netdev_get_numa_id(port->netdev);
2967
2968
dp_netdev_set_pmds_on_numa(dp, numa_id);
2969
}
2970
}
2971
}
2972
2973
static char *
2974
dpif_netdev_get_datapath_version(void)
2975
{
2976
return xstrdup("<built-in>");
2977
}
2978
2979
static void
2980
dp_netdev_flow_used(struct dp_netdev_flow *netdev_flow, int cnt, int size,
2981
uint16_t tcp_flags, long long now)
2982
{
2983
uint16_t flags;
2984
2985
atomic_store_relaxed(&netdev_flow->stats.used, now);
2986
non_atomic_ullong_add(&netdev_flow->stats.packet_count, cnt);
2987
non_atomic_ullong_add(&netdev_flow->stats.byte_count, size);
2988
atomic_read_relaxed(&netdev_flow->stats.tcp_flags, &flags);
2989
flags |= tcp_flags;
2990
atomic_store_relaxed(&netdev_flow->stats.tcp_flags, flags);
2991
}
2992
2993
static void
2994
dp_netdev_count_packet(struct dp_netdev_pmd_thread *pmd,
2995
enum dp_stat_type type, int cnt)
2996
{
2997
non_atomic_ullong_add(&pmd->stats.n[type], cnt);
2998
}
2999
3000
static int
3001
dp_netdev_upcall(struct dp_netdev_pmd_thread *pmd, struct dp_packet *packet_,
3002
struct flow *flow, struct flow_wildcards *wc, ovs_u128 *ufid,
3003
enum dpif_upcall_type type, const struct nlattr *userdata,
3004
struct ofpbuf *actions, struct ofpbuf *put_actions)
3005
{
3006
struct dp_netdev *dp = pmd->dp;
3007
3008
if (OVS_UNLIKELY(!dp->upcall_cb)) {
3009
return ENODEV;
3010
}
3011
3012
if (OVS_UNLIKELY(!VLOG_DROP_DBG(&upcall_rl))) {
3013
struct ds ds = DS_EMPTY_INITIALIZER;
3014
char *packet_str;
3015
struct ofpbuf key;
3016
3017
ofpbuf_init(&key, 0);
3018
odp_flow_key_from_flow(&key, flow, &wc->masks, flow->in_port.odp_port,
3019
true);
3020
packet_str = ofp_packet_to_string(dp_packet_data(packet_),
3021
dp_packet_size(packet_));
3022
3023
odp_flow_key_format(key.data, key.size, &ds);
3024
3025
VLOG_DBG("%s: %s upcall:\n%s\n%s", dp->name,
3026
dpif_upcall_type_to_string(type), ds_cstr(&ds), packet_str);
3027
3028
ofpbuf_uninit(&key);
3029
free(packet_str);
3030
3031
ds_destroy(&ds);
3032
}
3033
3034
return dp->upcall_cb(packet_, flow, ufid, pmd->core_id, type, userdata,
3035
actions, wc, put_actions, dp->upcall_aux);
3036
}
3037
3038
static inline uint32_t
3039
dpif_netdev_packet_get_rss_hash(struct dp_packet *packet,
3040
const struct miniflow *mf)
3041
{
3042
uint32_t hash, recirc_depth;
3043
3044
hash = dp_packet_get_rss_hash(packet);
3045
if (OVS_UNLIKELY(!hash)) {
3046
hash = miniflow_hash_5tuple(mf, 0);
3047
dp_packet_set_rss_hash(packet, hash);
3048
}
3049
3050
/* The RSS hash must account for the recirculation depth to avoid
3051
* collisions in the exact match cache */
3052
recirc_depth = *recirc_depth_get_unsafe();
3053
if (OVS_UNLIKELY(recirc_depth)) {
3054
hash = hash_finish(hash, recirc_depth);
3055
dp_packet_set_rss_hash(packet, hash);
3056
}
3057
return hash;
3058
}
3059
3060
struct packet_batch {
3061
unsigned int packet_count;
3062
unsigned int byte_count;
3063
uint16_t tcp_flags;
3064
3065
struct dp_netdev_flow *flow;
3066
3067
struct dp_packet *packets[NETDEV_MAX_BURST];
3068
};
3069
3070
static inline void
3071
packet_batch_update(struct packet_batch *batch, struct dp_packet *packet,
3072
const struct miniflow *mf)
3073
{
3074
batch->tcp_flags |= miniflow_get_tcp_flags(mf);
3075
batch->packets[batch->packet_count++] = packet;
3076
batch->byte_count += dp_packet_size(packet);
3077
}
3078
3079
static inline void
3080
packet_batch_init(struct packet_batch *batch, struct dp_netdev_flow *flow)
3081
{
3082
flow->batch = batch;
3083
3084
batch->flow = flow;
3085
batch->packet_count = 0;
3086
batch->byte_count = 0;
3087
batch->tcp_flags = 0;
3088
}
3089
3090
static inline void
3091
packet_batch_execute(struct packet_batch *batch,
3092
struct dp_netdev_pmd_thread *pmd,
3093
long long now)
3094
{
3095
struct dp_netdev_actions *actions;
3096
struct dp_netdev_flow *flow = batch->flow;
3097
3098
dp_netdev_flow_used(flow, batch->packet_count, batch->byte_count,
3099
batch->tcp_flags, now);
3100
3101
actions = dp_netdev_flow_get_actions(flow);
3102
3103
dp_netdev_execute_actions(pmd, batch->packets, batch->packet_count, true,
3104
actions->actions, actions->size);
3105
}
3106
3107
static inline void
3108
dp_netdev_queue_batches(struct dp_packet *pkt,
3109
struct dp_netdev_flow *flow, const struct miniflow *mf,
3110
struct packet_batch *batches, size_t *n_batches)
3111
{
3112
struct packet_batch *batch = flow->batch;
3113
3114
if (OVS_LIKELY(batch)) {
3115
packet_batch_update(batch, pkt, mf);
2027
3116
return;
2028
3117
}
2029
miniflow_initialize(&key.flow, key.buf);
2030
miniflow_extract(packet, md, &key.flow);
2031
2032
netdev_flow = dp_netdev_lookup_flow(dp, &key.flow);
2033
if (netdev_flow) {
2034
struct dp_netdev_actions *actions;
2035
2036
dp_netdev_flow_used(netdev_flow, packet, &key.flow);
2037
2038
actions = dp_netdev_flow_get_actions(netdev_flow);
2039
dp_netdev_execute_actions(dp, &key.flow, packet, true, md,
2040
actions->actions, actions->size);
2041
dp_netdev_count_packet(dp, DP_STAT_HIT);
2042
} else if (dp->handler_queues) {
2043
dp_netdev_count_packet(dp, DP_STAT_MISS);
2044
dp_netdev_output_userspace(dp, packet,
2045
miniflow_hash_5tuple(&key.flow, 0)
2046
% dp->n_handlers,
2047
DPIF_UC_MISS, &key.flow, NULL);
2048
ofpbuf_delete(packet);
2049
}
3118
3119
batch = &batches[(*n_batches)++];
3120
packet_batch_init(batch, flow);
3121
packet_batch_update(batch, pkt, mf);
3122
}
3123
3124
static inline void
3125
dp_packet_swap(struct dp_packet **a, struct dp_packet **b)
3126
{
3127
struct dp_packet *tmp = *a;
3128
*a = *b;
3129
*b = tmp;
3130
}
3131
3132
/* Try to process all ('cnt') the 'packets' using only the exact match cache
3133
* 'flow_cache'. If a flow is not found for a packet 'packets[i]', the
3134
* miniflow is copied into 'keys' and the packet pointer is moved at the
3135
* beginning of the 'packets' array.
3136
*
3137
* The function returns the number of packets that needs to be processed in the
3138
* 'packets' array (they have been moved to the beginning of the vector).
3139
*/
3140
static inline size_t
3141
emc_processing(struct dp_netdev_pmd_thread *pmd, struct dp_packet **packets,
3142
size_t cnt, struct netdev_flow_key *keys,
3143
struct packet_batch batches[], size_t *n_batches)
3144
{
3145
struct emc_cache *flow_cache = &pmd->flow_cache;
3146
struct netdev_flow_key key;
3147
size_t i, notfound_cnt = 0;
3148
3149
miniflow_initialize(&key.mf, key.buf);
3150
for (i = 0; i < cnt; i++) {
3151
struct dp_netdev_flow *flow;
3152
3153
if (OVS_UNLIKELY(dp_packet_size(packets[i]) < ETH_HEADER_LEN)) {
3154
dp_packet_delete(packets[i]);
3155
continue;
3156
}
3157
3158
if (i != cnt - 1) {
3159
/* Prefetch next packet data */
3160
OVS_PREFETCH(dp_packet_data(packets[i+1]));
3161
}
3162
3163
miniflow_extract(packets[i], &key.mf);
3164
key.len = 0; /* Not computed yet. */
3165
key.hash = dpif_netdev_packet_get_rss_hash(packets[i], &key.mf);
3166
3167
flow = emc_lookup(flow_cache, &key);
3168
if (OVS_LIKELY(flow)) {
3169
dp_netdev_queue_batches(packets[i], flow, &key.mf, batches,
3170
n_batches);
3171
} else {
3172
if (i != notfound_cnt) {
3173
dp_packet_swap(&packets[i], &packets[notfound_cnt]);
3174
}
3175
3176
keys[notfound_cnt++] = key;
3177
}
3178
}
3179
3180
dp_netdev_count_packet(pmd, DP_STAT_EXACT_HIT, cnt - notfound_cnt);
3181
3182
return notfound_cnt;
3183
}
3184
3185
static inline void
3186
fast_path_processing(struct dp_netdev_pmd_thread *pmd,
3187
struct dp_packet **packets, size_t cnt,
3188
struct netdev_flow_key *keys,
3189
struct packet_batch batches[], size_t *n_batches)
3190
{
3191
#if !defined(__CHECKER__) && !defined(_WIN32)
3192
const size_t PKT_ARRAY_SIZE = cnt;
3193
#else
3194
/* Sparse or MSVC doesn't like variable length array. */
3195
enum { PKT_ARRAY_SIZE = NETDEV_MAX_BURST };
3196
#endif
3197
struct dpcls_rule *rules[PKT_ARRAY_SIZE];
3198
struct dp_netdev *dp = pmd->dp;
3199
struct emc_cache *flow_cache = &pmd->flow_cache;
3200
int miss_cnt = 0, lost_cnt = 0;
3201
bool any_miss;
3202
size_t i;
3203
3204
for (i = 0; i < cnt; i++) {
3205
/* Key length is needed in all the cases, hash computed on demand. */
3206
keys[i].len = netdev_flow_key_size(count_1bits(keys[i].mf.map));
3207
}
3208
any_miss = !dpcls_lookup(&pmd->cls, keys, rules, cnt);
3209
if (OVS_UNLIKELY(any_miss) && !fat_rwlock_tryrdlock(&dp->upcall_rwlock)) {
3210
uint64_t actions_stub[512 / 8], slow_stub[512 / 8];
3211
struct ofpbuf actions, put_actions;
3212
ovs_u128 ufid;
3213
3214
ofpbuf_use_stub(&actions, actions_stub, sizeof actions_stub);
3215
ofpbuf_use_stub(&put_actions, slow_stub, sizeof slow_stub);
3216
3217
for (i = 0; i < cnt; i++) {
3218
struct dp_netdev_flow *netdev_flow;
3219
struct ofpbuf *add_actions;
3220
struct match match;
3221
int error;
3222
3223
if (OVS_LIKELY(rules[i])) {
3224
continue;
3225
}
3226
3227
/* It's possible that an earlier slow path execution installed
3228
* a rule covering this flow. In this case, it's a lot cheaper
3229
* to catch it here than execute a miss. */
3230
netdev_flow = dp_netdev_pmd_lookup_flow(pmd, &keys[i]);
3231
if (netdev_flow) {
3232
rules[i] = &netdev_flow->cr;
3233
continue;
3234
}
3235
3236
miss_cnt++;
3237
3238
miniflow_expand(&keys[i].mf, &match.flow);
3239
3240
ofpbuf_clear(&actions);
3241
ofpbuf_clear(&put_actions);
3242
3243
dpif_flow_hash(dp->dpif, &match.flow, sizeof match.flow, &ufid);
3244
error = dp_netdev_upcall(pmd, packets[i], &match.flow, &match.wc,
3245
&ufid, DPIF_UC_MISS, NULL, &actions,
3246
&put_actions);
3247
if (OVS_UNLIKELY(error && error != ENOSPC)) {
3248
dp_packet_delete(packets[i]);
3249
lost_cnt++;
3250
continue;
3251
}
3252
3253
/* We can't allow the packet batching in the next loop to execute
3254
* the actions. Otherwise, if there are any slow path actions,
3255
* we'll send the packet up twice. */
3256
dp_netdev_execute_actions(pmd, &packets[i], 1, true,
3257
actions.data, actions.size);
3258
3259
add_actions = put_actions.size ? &put_actions : &actions;
3260
if (OVS_LIKELY(error != ENOSPC)) {
3261
/* XXX: There's a race window where a flow covering this packet
3262
* could have already been installed since we last did the flow
3263
* lookup before upcall. This could be solved by moving the
3264
* mutex lock outside the loop, but that's an awful long time
3265
* to be locking everyone out of making flow installs. If we
3266
* move to a per-core classifier, it would be reasonable. */
3267
ovs_mutex_lock(&pmd->flow_mutex);
3268
netdev_flow = dp_netdev_pmd_lookup_flow(pmd, &keys[i]);
3269
if (OVS_LIKELY(!netdev_flow)) {
3270
netdev_flow = dp_netdev_flow_add(pmd, &match, &ufid,
3271
add_actions->data,
3272
add_actions->size);
3273
}
3274
ovs_mutex_unlock(&pmd->flow_mutex);
3275
3276
emc_insert(flow_cache, &keys[i], netdev_flow);
3277
}
3278
}
3279
3280
ofpbuf_uninit(&actions);
3281
ofpbuf_uninit(&put_actions);
3282
fat_rwlock_unlock(&dp->upcall_rwlock);
3283
dp_netdev_count_packet(pmd, DP_STAT_LOST, lost_cnt);
3284
} else if (OVS_UNLIKELY(any_miss)) {
3285
for (i = 0; i < cnt; i++) {
3286
if (OVS_UNLIKELY(!rules[i])) {
3287
dp_packet_delete(packets[i]);
3288
lost_cnt++;
3289
miss_cnt++;
3290
}
3291
}
3292
}
3293
3294
for (i = 0; i < cnt; i++) {
3295
struct dp_packet *packet = packets[i];
3296
struct dp_netdev_flow *flow;
3297
3298
if (OVS_UNLIKELY(!rules[i])) {
3299
continue;
3300
}
3301
3302
flow = dp_netdev_flow_cast(rules[i]);
3303
3304
emc_insert(flow_cache, &keys[i], flow);
3305
dp_netdev_queue_batches(packet, flow, &keys[i].mf, batches, n_batches);
3306
}
3307
3308
dp_netdev_count_packet(pmd, DP_STAT_MASKED_HIT, cnt - miss_cnt);
3309
dp_netdev_count_packet(pmd, DP_STAT_MISS, miss_cnt);
3310
dp_netdev_count_packet(pmd, DP_STAT_LOST, lost_cnt);
2050
3311
}
2051
3312
2052
3313
static void
2053
dp_netdev_port_input(struct dp_netdev *dp, struct ofpbuf *packet,
2054
struct pkt_metadata *md)
2055
OVS_REQ_RDLOCK(dp->port_rwlock)
2056
{
2057
uint32_t *recirc_depth = recirc_depth_get();
2058
2059
*recirc_depth = 0;
2060
dp_netdev_input(dp, packet, md);
2061
}
2062
2063
static int
2064
dp_netdev_output_userspace(struct dp_netdev *dp, struct ofpbuf *packet,
2065
int queue_no, int type, const struct miniflow *key,
2066
const struct nlattr *userdata)
2067
{
2068
struct dp_netdev_queue *q;
2069
int error;
2070
2071
fat_rwlock_rdlock(&dp->queue_rwlock);
2072
q = &dp->handler_queues[queue_no];
2073
ovs_mutex_lock(&q->mutex);
2074
if (q->head - q->tail < MAX_QUEUE_LEN) {
2075
struct dp_netdev_upcall *u = &q->upcalls[q->head++ & QUEUE_MASK];
2076
struct dpif_upcall *upcall = &u->upcall;
2077
struct ofpbuf *buf = &u->buf;
2078
size_t buf_size;
2079
struct flow flow;
2080
void *data;
2081
2082
upcall->type = type;
2083
2084
/* Allocate buffer big enough for everything. */
2085
buf_size = ODPUTIL_FLOW_KEY_BYTES;
2086
if (userdata) {
2087
buf_size += NLA_ALIGN(userdata->nla_len);
2088
}
2089
buf_size += ofpbuf_size(packet);
2090
ofpbuf_init(buf, buf_size);
2091
2092
/* Put ODP flow. */
2093
miniflow_expand(key, &flow);
2094
odp_flow_key_from_flow(buf, &flow, NULL, flow.in_port.odp_port);
2095
upcall->key = ofpbuf_data(buf);
2096
upcall->key_len = ofpbuf_size(buf);
2097
2098
/* Put userdata. */
2099
if (userdata) {
2100
upcall->userdata = ofpbuf_put(buf, userdata,
2101
NLA_ALIGN(userdata->nla_len));
2102
}
2103
2104
data = ofpbuf_put(buf, ofpbuf_data(packet), ofpbuf_size(packet));
2105
ofpbuf_use_stub(&upcall->packet, data, ofpbuf_size(packet));
2106
ofpbuf_set_size(&upcall->packet, ofpbuf_size(packet));
2107
2108
seq_change(q->seq);
2109
2110
error = 0;
2111
} else {
2112
dp_netdev_count_packet(dp, DP_STAT_LOST);
2113
error = ENOBUFS;
2114
}
2115
ovs_mutex_unlock(&q->mutex);
2116
fat_rwlock_unlock(&dp->queue_rwlock);
2117
2118
return error;
3314
dp_netdev_input(struct dp_netdev_pmd_thread *pmd,
3315
struct dp_packet **packets, int cnt)
3316
{
3317
#if !defined(__CHECKER__) && !defined(_WIN32)
3318
const size_t PKT_ARRAY_SIZE = cnt;
3319
#else
3320
/* Sparse or MSVC doesn't like variable length array. */
3321
enum { PKT_ARRAY_SIZE = NETDEV_MAX_BURST };
3322
#endif
3323
struct netdev_flow_key keys[PKT_ARRAY_SIZE];
3324
struct packet_batch batches[PKT_ARRAY_SIZE];
3325
long long now = time_msec();
3326
size_t newcnt, n_batches, i;
3327
3328
n_batches = 0;
3329
newcnt = emc_processing(pmd, packets, cnt, keys, batches, &n_batches);
3330
if (OVS_UNLIKELY(newcnt)) {
3331
fast_path_processing(pmd, packets, newcnt, keys, batches, &n_batches);
3332
}
3333
3334
for (i = 0; i < n_batches; i++) {
3335
batches[i].flow->batch = NULL;
3336
}
3337
3338
for (i = 0; i < n_batches; i++) {
3339
packet_batch_execute(&batches[i], pmd, now);
3340
}
2119
3341
}
2120
3342
2121
3343
struct dp_netdev_execute_aux {
2122
struct dp_netdev *dp;
2123
const struct miniflow *key;
3344
struct dp_netdev_pmd_thread *pmd;
2124
3345
};
2125
3346
2126
3347
static void
2127
dp_execute_cb(void *aux_, struct ofpbuf *packet,
2128
struct pkt_metadata *md,
3348
dpif_netdev_register_upcall_cb(struct dpif *dpif, upcall_callback *cb,
3349
void *aux)
3350
{
3351
struct dp_netdev *dp = get_dp_netdev(dpif);
3352
dp->upcall_aux = aux;
3353
dp->upcall_cb = cb;
3354
}
3355
3356
static void
3357
dp_netdev_drop_packets(struct dp_packet **packets, int cnt, bool may_steal)
3358
{
3359
if (may_steal) {
3360
int i;
3361
3362
for (i = 0; i < cnt; i++) {
3363
dp_packet_delete(packets[i]);
3364
}
3365
}
3366
}
3367
3368
static int
3369
push_tnl_action(const struct dp_netdev *dp,
3370
const struct nlattr *attr,
3371
struct dp_packet **packets, int cnt)
3372
{
3373
struct dp_netdev_port *tun_port;
3374
const struct ovs_action_push_tnl *data;
3375
3376
data = nl_attr_get(attr);
3377
3378
tun_port = dp_netdev_lookup_port(dp, u32_to_odp(data->tnl_port));
3379
if (!tun_port) {
3380
return -EINVAL;
3381
}
3382
netdev_push_header(tun_port->netdev, packets, cnt, data);
3383
3384
return 0;
3385
}
3386
3387
static void
3388
dp_netdev_clone_pkt_batch(struct dp_packet **dst_pkts,
3389
struct dp_packet **src_pkts, int cnt)
3390
{
3391
int i;
3392
3393
for (i = 0; i < cnt; i++) {
3394
dst_pkts[i] = dp_packet_clone(src_pkts[i]);
3395
}
3396
}
3397
3398
static void
3399
dp_execute_cb(void *aux_, struct dp_packet **packets, int cnt,
2129
3400
const struct nlattr *a, bool may_steal)
2130
3401
OVS_NO_THREAD_SAFETY_ANALYSIS
2131
3402
{
2132
3403
struct dp_netdev_execute_aux *aux = aux_;
3404
uint32_t *depth = recirc_depth_get();
3405
struct dp_netdev_pmd_thread *pmd = aux->pmd;
3406
struct dp_netdev *dp = pmd->dp;
2133
3407
int type = nl_attr_type(a);
2134
3408
struct dp_netdev_port *p;
2135
uint32_t *depth = recirc_depth_get();
3409
int i;
2136
3410
2137
3411
switch ((enum ovs_action_attr)type) {
2138
3412
case OVS_ACTION_ATTR_OUTPUT:
2139
p = dp_netdev_lookup_port(aux->dp, u32_to_odp(nl_attr_get_u32(a)));
2140
if (p) {
2141
netdev_send(p->netdev, packet, may_steal);
2142
} else if (may_steal) {
2143
ofpbuf_delete(packet);
2144
}
2145
2146
break;
2147
2148
case OVS_ACTION_ATTR_USERSPACE: {
2149
const struct nlattr *userdata;
2150
2151
userdata = nl_attr_find_nested(a, OVS_USERSPACE_ATTR_USERDATA);
2152
2153
if (aux->dp->n_handlers > 0) {
2154
dp_netdev_output_userspace(aux->dp, packet,
2155
miniflow_hash_5tuple(aux->key, 0)
2156
% aux->dp->n_handlers,
2157
DPIF_UC_ACTION, aux->key,
2158
userdata);
2159
}
2160
2161
if (may_steal) {
2162
ofpbuf_delete(packet);
2163
}
2164
break;
2165
}
2166
2167
case OVS_ACTION_ATTR_HASH: {
2168
const struct ovs_action_hash *hash_act;
2169
uint32_t hash;
2170
2171
hash_act = nl_attr_get(a);
2172
if (hash_act->hash_alg == OVS_HASH_ALG_L4) {
2173
/* Hash need not be symmetric, nor does it need to include
2174
* L2 fields. */
2175
hash = miniflow_hash_5tuple(aux->key, hash_act->hash_basis);
2176
if (!hash) {
2177
hash = 1; /* 0 is not valid */
2178
}
2179
2180
} else {
2181
VLOG_WARN("Unknown hash algorithm specified for the hash action.");
2182
hash = 2;
2183
}
2184
2185
md->dp_hash = hash;
2186
break;
2187
}
3413
p = dp_netdev_lookup_port(dp, u32_to_odp(nl_attr_get_u32(a)));
3414
if (OVS_LIKELY(p)) {
3415
netdev_send(p->netdev, pmd->tx_qid, packets, cnt, may_steal);
3416
return;
3417
}
3418
break;
3419
3420
case OVS_ACTION_ATTR_TUNNEL_PUSH:
3421
if (*depth < MAX_RECIRC_DEPTH) {
3422
struct dp_packet *tnl_pkt[NETDEV_MAX_BURST];
3423
int err;
3424
3425
if (!may_steal) {
3426
dp_netdev_clone_pkt_batch(tnl_pkt, packets, cnt);
3427
packets = tnl_pkt;
3428
}
3429
3430
err = push_tnl_action(dp, a, packets, cnt);
3431
if (!err) {
3432
(*depth)++;
3433
dp_netdev_input(pmd, packets, cnt);
3434
(*depth)--;
3435
} else {
3436
dp_netdev_drop_packets(tnl_pkt, cnt, !may_steal);
3437
}
3438
return;
3439
}
3440
break;
3441
3442
case OVS_ACTION_ATTR_TUNNEL_POP:
3443
if (*depth < MAX_RECIRC_DEPTH) {
3444
odp_port_t portno = u32_to_odp(nl_attr_get_u32(a));
3445
3446
p = dp_netdev_lookup_port(dp, portno);
3447
if (p) {
3448
struct dp_packet *tnl_pkt[NETDEV_MAX_BURST];
3449
int err;
3450
3451
if (!may_steal) {
3452
dp_netdev_clone_pkt_batch(tnl_pkt, packets, cnt);
3453
packets = tnl_pkt;
3454
}
3455
3456
err = netdev_pop_header(p->netdev, packets, cnt);
3457
if (!err) {
3458
3459
for (i = 0; i < cnt; i++) {
3460
packets[i]->md.in_port.odp_port = portno;
3461
}
3462
3463
(*depth)++;
3464
dp_netdev_input(pmd, packets, cnt);
3465
(*depth)--;
3466
} else {
3467
dp_netdev_drop_packets(tnl_pkt, cnt, !may_steal);
3468
}
3469
return;
3470
}
3471
}
3472
break;
3473
3474
case OVS_ACTION_ATTR_USERSPACE:
3475
if (!fat_rwlock_tryrdlock(&dp->upcall_rwlock)) {
3476
const struct nlattr *userdata;
3477
struct ofpbuf actions;
3478
struct flow flow;
3479
ovs_u128 ufid;
3480
3481
userdata = nl_attr_find_nested(a, OVS_USERSPACE_ATTR_USERDATA);
3482
ofpbuf_init(&actions, 0);
3483
3484
for (i = 0; i < cnt; i++) {
3485
int error;
3486
3487
ofpbuf_clear(&actions);
3488
3489
flow_extract(packets[i], &flow);
3490
dpif_flow_hash(dp->dpif, &flow, sizeof flow, &ufid);
3491
error = dp_netdev_upcall(pmd, packets[i], &flow, NULL, &ufid,
3492
DPIF_UC_ACTION, userdata,&actions,
3493
NULL);
3494
if (!error || error == ENOSPC) {
3495
dp_netdev_execute_actions(pmd, &packets[i], 1, may_steal,
3496
actions.data, actions.size);
3497
} else if (may_steal) {
3498
dp_packet_delete(packets[i]);
3499
}
3500
}
3501
ofpbuf_uninit(&actions);
3502
fat_rwlock_unlock(&dp->upcall_rwlock);
3503
3504
return;
3505
}
3506
break;
2188
3507
2189
3508
case OVS_ACTION_ATTR_RECIRC:
2190
3509
if (*depth < MAX_RECIRC_DEPTH) {
2191
struct pkt_metadata recirc_md = *md;
2192
struct ofpbuf *recirc_packet;
2193
2194
recirc_packet = may_steal ? packet : ofpbuf_clone(packet);
2195
recirc_md.recirc_id = nl_attr_get_u32(a);
3510
struct dp_packet *recirc_pkts[NETDEV_MAX_BURST];
3511
3512
if (!may_steal) {
3513
dp_netdev_clone_pkt_batch(recirc_pkts, packets, cnt);
3514
packets = recirc_pkts;
3515
}
3516
3517
for (i = 0; i < cnt; i++) {
3518
packets[i]->md.recirc_id = nl_attr_get_u32(a);
3519
}
2196
3520
2197
3521
(*depth)++;
2198
dp_netdev_input(aux->dp, recirc_packet, &recirc_md);
3522
dp_netdev_input(pmd, packets, cnt);
2199
3523
(*depth)--;
2200
3524
2201
break;
2202
} else {
2203
if (may_steal) {
2204
ofpbuf_delete(packet);
2205
}
2206
VLOG_WARN("Packet dropped. Max recirculation depth exceeded.");
3525
return;
2207
3526
}
3527
3528
VLOG_WARN("Packet dropped. Max recirculation depth exceeded.");
2208
3529
break;
2209
3530
2210
3531
case OVS_ACTION_ATTR_PUSH_VLAN:
2212
3533
case OVS_ACTION_ATTR_PUSH_MPLS:
2213
3534
case OVS_ACTION_ATTR_POP_MPLS:
2214
3535
case OVS_ACTION_ATTR_SET:
3536
case OVS_ACTION_ATTR_SET_MASKED:
2215
3537
case OVS_ACTION_ATTR_SAMPLE:
3538
case OVS_ACTION_ATTR_HASH:
2216
3539
case OVS_ACTION_ATTR_UNSPEC:
2217
3540
case __OVS_ACTION_ATTR_MAX:
2218
3541
OVS_NOT_REACHED();
2219
3542
}
3543
3544
dp_netdev_drop_packets(packets, cnt, may_steal);
2220
3545
}
2221
3546
2222
3547
static void
2223
dp_netdev_execute_actions(struct dp_netdev *dp, const struct miniflow *key,
2224
struct ofpbuf *packet, bool may_steal,
2225
struct pkt_metadata *md,
3548
dp_netdev_execute_actions(struct dp_netdev_pmd_thread *pmd,
3549
struct dp_packet **packets, int cnt,
3550
bool may_steal,
2226
3551
const struct nlattr *actions, size_t actions_len)
2227
3552
{
2228
struct dp_netdev_execute_aux aux = {dp, key};
3553
struct dp_netdev_execute_aux aux = { pmd };
2229
3554
2230
odp_execute_actions(&aux, packet, may_steal, md,
2231
actions, actions_len, dp_execute_cb);
3555
odp_execute_actions(&aux, packets, cnt, may_steal, actions,
3556
actions_len, dp_execute_cb);
2232
3557
}
2233
3558
2234
3559
const struct dpif_class dpif_netdev_class = {
2235
3560
"netdev",
3561
dpif_netdev_init,
2236
3562
dpif_netdev_enumerate,
2237
3563
dpif_netdev_port_open_type,
2238
3564
dpif_netdev_open,
2251
3577
dpif_netdev_port_dump_done,
2252
3578
dpif_netdev_port_poll,
2253
3579
dpif_netdev_port_poll_wait,
2254
dpif_netdev_flow_get,
2255
dpif_netdev_flow_put,
2256
dpif_netdev_flow_del,
2257
3580
dpif_netdev_flow_flush,
2258
dpif_netdev_flow_dump_state_init,
2259
dpif_netdev_flow_dump_start,
3581
dpif_netdev_flow_dump_create,
3582
dpif_netdev_flow_dump_destroy,
3583
dpif_netdev_flow_dump_thread_create,
3584
dpif_netdev_flow_dump_thread_destroy,
2260
3585
dpif_netdev_flow_dump_next,
2261
NULL,
2262
dpif_netdev_flow_dump_done,
2263
dpif_netdev_flow_dump_state_uninit,
2264
dpif_netdev_execute,
2265
NULL, /* operate */
2266
dpif_netdev_recv_set,
2267
dpif_netdev_handlers_set,
3586
dpif_netdev_operate,
3587
NULL, /* recv_set */
3588
NULL, /* handlers_set */
3589
dpif_netdev_pmd_set,
2268
3590
dpif_netdev_queue_to_priority,
2269
dpif_netdev_recv,
2270
dpif_netdev_recv_wait,
2271
dpif_netdev_recv_purge,
3591
NULL, /* recv */
3592
NULL, /* recv_wait */
3593
NULL, /* recv_purge */
3594
dpif_netdev_register_upcall_cb,
3595
dpif_netdev_enable_upcall,
3596
dpif_netdev_disable_upcall,
3597
dpif_netdev_get_datapath_version,
2272
3598
};
2273
3599
2274
3600
static void
2275
3601
dpif_dummy_change_port_number(struct unixctl_conn *conn, int argc OVS_UNUSED,
2276
3602
const char *argv[], void *aux OVS_UNUSED)
2277
3603
{
2278
struct dp_netdev_port *port;
3604
struct dp_netdev_port *old_port;
3605
struct dp_netdev_port *new_port;
2279
3606
struct dp_netdev *dp;
2280
3607
odp_port_t port_no;
2281
3608
2289
3616
ovs_refcount_ref(&dp->ref_cnt);
2290
3617
ovs_mutex_unlock(&dp_netdev_mutex);
2291
3618
2292
ovs_rwlock_wrlock(&dp->port_rwlock);
2293
if (get_port_by_name(dp, argv[2], &port)) {
3619
ovs_mutex_lock(&dp->port_mutex);
3620
if (get_port_by_name(dp, argv[2], &old_port)) {
2294
3621
unixctl_command_reply_error(conn, "unknown port");
2295
3622
goto exit;
2296
3623
}
2304
3631
unixctl_command_reply_error(conn, "port number already in use");
2305
3632
goto exit;
2306
3633
}
2307
hmap_remove(&dp->ports, &port->node);
2308
port->port_no = port_no;
2309
hmap_insert(&dp->ports, &port->node, hash_int(odp_to_u32(port_no), 0));
3634
3635
/* Remove old port. */
3636
cmap_remove(&dp->ports, &old_port->node, hash_port_no(old_port->md.in_port.odp_port));
3637
ovsrcu_postpone(free, old_port);
3638
3639
/* Insert new port (cmap semantics mean we cannot re-insert 'old_port'). */
3640
new_port = xmemdup(old_port, sizeof *old_port);
3641
new_port->md.in_port.odp_port = port_no;
3642
cmap_insert(&dp->ports, &new_port->node, hash_port_no(port_no));
3643
2310
3644
seq_change(dp->port_seq);
2311
3645
unixctl_command_reply(conn, NULL);
2312
3646
2313
3647
exit:
2314
ovs_rwlock_unlock(&dp->port_rwlock);
3648
ovs_mutex_unlock(&dp->port_mutex);
3649
dp_netdev_unref(dp);
3650
}
3651
3652
static void
3653
dpif_dummy_delete_port(struct unixctl_conn *conn, int argc OVS_UNUSED,
3654
const char *argv[], void *aux OVS_UNUSED)
3655
{
3656
struct dp_netdev_port *port;
3657
struct dp_netdev *dp;
3658
3659
ovs_mutex_lock(&dp_netdev_mutex);
3660
dp = shash_find_data(&dp_netdevs, argv[1]);
3661
if (!dp || !dpif_netdev_class_is_dummy(dp->class)) {
3662
ovs_mutex_unlock(&dp_netdev_mutex);
3663
unixctl_command_reply_error(conn, "unknown datapath or not a dummy");
3664
return;
3665
}
3666
ovs_refcount_ref(&dp->ref_cnt);
3667
ovs_mutex_unlock(&dp_netdev_mutex);
3668
3669
ovs_mutex_lock(&dp->port_mutex);
3670
if (get_port_by_name(dp, argv[2], &port)) {
3671
unixctl_command_reply_error(conn, "unknown port");
3672
} else if (port->md.in_port.odp_port == ODPP_LOCAL) {
3673
unixctl_command_reply_error(conn, "can't delete local port");
3674
} else {
3675
do_del_port(dp, port);
3676
unixctl_command_reply(conn, NULL);
3677
}
3678
ovs_mutex_unlock(&dp->port_mutex);
3679
2315
3680
dp_netdev_unref(dp);
2316
3681
}
2317
3682
2346
3711
dpif_dummy_register__("dummy");
2347
3712
2348
3713
unixctl_command_register("dpif-dummy/change-port-number",
2349
"DP PORT NEW-NUMBER",
3714
"dp port new-number",
2350
3715
3, 3, dpif_dummy_change_port_number, NULL);
3716
unixctl_command_register("dpif-dummy/delete-port", "dp port",
3717
2, 2, dpif_dummy_delete_port, NULL);
3718
}
3719
3720
/* Datapath Classifier. */
3721
3722
/* A set of rules that all have the same fields wildcarded. */
3723
struct dpcls_subtable {
3724
/* The fields are only used by writers. */
3725
struct cmap_node cmap_node OVS_GUARDED; /* Within dpcls 'subtables_map'. */
3726
3727
/* These fields are accessed by readers. */
3728
struct cmap rules; /* Contains "struct dpcls_rule"s. */
3729
struct netdev_flow_key mask; /* Wildcards for fields (const). */
3730
/* 'mask' must be the last field, additional space is allocated here. */
3731
};
3732
3733
/* Initializes 'cls' as a classifier that initially contains no classification
3734
* rules. */
3735
static void
3736
dpcls_init(struct dpcls *cls)
3737
{
3738
cmap_init(&cls->subtables_map);
3739
pvector_init(&cls->subtables);
3740
}
3741
3742
static void
3743
dpcls_destroy_subtable(struct dpcls *cls, struct dpcls_subtable *subtable)
3744
{
3745
pvector_remove(&cls->subtables, subtable);
3746
cmap_remove(&cls->subtables_map, &subtable->cmap_node,
3747
subtable->mask.hash);
3748
cmap_destroy(&subtable->rules);
3749
ovsrcu_postpone(free, subtable);
3750
}
3751
3752
/* Destroys 'cls'. Rules within 'cls', if any, are not freed; this is the
3753
* caller's responsibility.
3754
* May only be called after all the readers have been terminated. */
3755
static void
3756
dpcls_destroy(struct dpcls *cls)
3757
{
3758
if (cls) {
3759
struct dpcls_subtable *subtable;
3760
3761
CMAP_FOR_EACH (subtable, cmap_node, &cls->subtables_map) {
3762
dpcls_destroy_subtable(cls, subtable);
3763
}
3764
cmap_destroy(&cls->subtables_map);
3765
pvector_destroy(&cls->subtables);
3766
}
3767
}
3768
3769
static struct dpcls_subtable *
3770
dpcls_create_subtable(struct dpcls *cls, const struct netdev_flow_key *mask)
3771
{
3772
struct dpcls_subtable *subtable;
3773
3774
/* Need to add one. */
3775
subtable = xmalloc(sizeof *subtable
3776
- sizeof subtable->mask.mf + mask->len);
3777
cmap_init(&subtable->rules);
3778
netdev_flow_key_clone(&subtable->mask, mask);
3779
cmap_insert(&cls->subtables_map, &subtable->cmap_node, mask->hash);
3780
pvector_insert(&cls->subtables, subtable, 0);
3781
pvector_publish(&cls->subtables);
3782
3783
return subtable;
3784
}
3785
3786
static inline struct dpcls_subtable *
3787
dpcls_find_subtable(struct dpcls *cls, const struct netdev_flow_key *mask)
3788
{
3789
struct dpcls_subtable *subtable;
3790
3791
CMAP_FOR_EACH_WITH_HASH (subtable, cmap_node, mask->hash,
3792
&cls->subtables_map) {
3793
if (netdev_flow_key_equal(&subtable->mask, mask)) {
3794
return subtable;
3795
}
3796
}
3797
return dpcls_create_subtable(cls, mask);
3798
}
3799
3800
/* Insert 'rule' into 'cls'. */
3801
static void
3802
dpcls_insert(struct dpcls *cls, struct dpcls_rule *rule,
3803
const struct netdev_flow_key *mask)
3804
{
3805
struct dpcls_subtable *subtable = dpcls_find_subtable(cls, mask);
3806
3807
rule->mask = &subtable->mask;
3808
cmap_insert(&subtable->rules, &rule->cmap_node, rule->flow.hash);
3809
}
3810
3811
/* Removes 'rule' from 'cls', also destructing the 'rule'. */
3812
static void
3813
dpcls_remove(struct dpcls *cls, struct dpcls_rule *rule)
3814
{
3815
struct dpcls_subtable *subtable;
3816
3817
ovs_assert(rule->mask);
3818
3819
INIT_CONTAINER(subtable, rule->mask, mask);
3820
3821
if (cmap_remove(&subtable->rules, &rule->cmap_node, rule->flow.hash)
3822
== 0) {
3823
dpcls_destroy_subtable(cls, subtable);
3824
pvector_publish(&cls->subtables);
3825
}
3826
}
3827
3828
/* Returns true if 'target' satisifies 'key' in 'mask', that is, if each 1-bit
3829
* in 'mask' the values in 'key' and 'target' are the same.
3830
*
3831
* Note: 'key' and 'mask' have the same mask, and 'key' is already masked. */
3832
static inline bool
3833
dpcls_rule_matches_key(const struct dpcls_rule *rule,
3834
const struct netdev_flow_key *target)
3835
{
3836
const uint64_t *keyp = rule->flow.mf.inline_values;
3837
const uint64_t *maskp = rule->mask->mf.inline_values;
3838
uint64_t target_u64;
3839
3840
NETDEV_FLOW_KEY_FOR_EACH_IN_MAP(target_u64, target, rule->flow.mf.map) {
3841
if (OVS_UNLIKELY((target_u64 & *maskp++) != *keyp++)) {
3842
return false;
3843
}
3844
}
3845
return true;
3846
}
3847
3848
/* For each miniflow in 'flows' performs a classifier lookup writing the result
3849
* into the corresponding slot in 'rules'. If a particular entry in 'flows' is
3850
* NULL it is skipped.
3851
*
3852
* This function is optimized for use in the userspace datapath and therefore
3853
* does not implement a lot of features available in the standard
3854
* classifier_lookup() function. Specifically, it does not implement
3855
* priorities, instead returning any rule which matches the flow.
3856
*
3857
* Returns true if all flows found a corresponding rule. */
3858
static bool
3859
dpcls_lookup(const struct dpcls *cls, const struct netdev_flow_key keys[],
3860
struct dpcls_rule **rules, const size_t cnt)
3861
{
3862
/* The batch size 16 was experimentally found faster than 8 or 32. */
3863
typedef uint16_t map_type;
3864
#define MAP_BITS (sizeof(map_type) * CHAR_BIT)
3865
3866
#if !defined(__CHECKER__) && !defined(_WIN32)
3867
const int N_MAPS = DIV_ROUND_UP(cnt, MAP_BITS);
3868
#else
3869
enum { N_MAPS = DIV_ROUND_UP(NETDEV_MAX_BURST, MAP_BITS) };
3870
#endif
3871
map_type maps[N_MAPS];
3872
struct dpcls_subtable *subtable;
3873
3874
memset(maps, 0xff, sizeof maps);
3875
if (cnt % MAP_BITS) {
3876
maps[N_MAPS - 1] >>= MAP_BITS - cnt % MAP_BITS; /* Clear extra bits. */
3877
}
3878
memset(rules, 0, cnt * sizeof *rules);
3879
3880
PVECTOR_FOR_EACH (subtable, &cls->subtables) {
3881
const struct netdev_flow_key *mkeys = keys;
3882
struct dpcls_rule **mrules = rules;
3883
map_type remains = 0;
3884
int m;
3885
3886
BUILD_ASSERT_DECL(sizeof remains == sizeof *maps);
3887
3888
for (m = 0; m < N_MAPS; m++, mkeys += MAP_BITS, mrules += MAP_BITS) {
3889
uint32_t hashes[MAP_BITS];
3890
const struct cmap_node *nodes[MAP_BITS];
3891
unsigned long map = maps[m];
3892
int i;
3893
3894
if (!map) {
3895
continue; /* Skip empty maps. */
3896
}
3897
3898
/* Compute hashes for the remaining keys. */
3899
ULONG_FOR_EACH_1(i, map) {
3900
hashes[i] = netdev_flow_key_hash_in_mask(&mkeys[i],
3901
&subtable->mask);
3902
}
3903
/* Lookup. */
3904
map = cmap_find_batch(&subtable->rules, map, hashes, nodes);
3905
/* Check results. */
3906
ULONG_FOR_EACH_1(i, map) {
3907
struct dpcls_rule *rule;
3908
3909
CMAP_NODE_FOR_EACH (rule, cmap_node, nodes[i]) {
3910
if (OVS_LIKELY(dpcls_rule_matches_key(rule, &mkeys[i]))) {
3911
mrules[i] = rule;
3912
goto next;
3913
}
3914
}
3915
ULONG_SET0(map, i); /* Did not match. */
3916
next:
3917
; /* Keep Sparse happy. */
3918
}
3919
maps[m] &= ~map; /* Clear the found rules. */
3920
remains |= maps[m];
3921
}
3922
if (!remains) {
3923
return true; /* All found. */
3924
}
3925
}
3926
return false; /* Some misses. */
2351
3927
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.1