Skip to content

Commit f019679

Browse files
Chris MiSaeed Mahameed
Chris Mi
authored and
Saeed Mahameed
committed
net/mlx5: E-switch, Remove dependency between sriov and eswitch mode
Currently, there are three eswitch modes, none, legacy and switchdev. None is the default mode. Remove redundant none mode as eswitch mode should always be either legacy mode or switchdev mode. With this patch, there are two behavior changes: 1. Legacy becomes the default mode. When querying eswitch mode using devlink, a valid mode is always returned. 2. When disabling sriov, the eswitch mode will not change, only vfs are unloaded. Signed-off-by: Chris Mi <[email protected]> Reviewed-by: Maor Dickman <[email protected]> Reviewed-by: Roi Dayan <[email protected]> Signed-off-by: Saeed Mahameed <[email protected]>
1 parent fbd43b7 commit f019679

File tree

8 files changed

+89
-83
lines changed

8 files changed

+89
-83
lines changed

‎drivers/net/ethernet/mellanox/mlx5/core/eswitch.c

Lines changed: 54 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,8 +1152,6 @@ mlx5_eswitch_update_num_of_vfs(struct mlx5_eswitch *esw, int num_vfs)
11521152
{
11531153
const u32 *out;
11541154

1155-
WARN_ON_ONCE(esw->mode != MLX5_ESWITCH_NONE);
1156-
11571155
if (num_vfs < 0)
11581156
return;
11591157

@@ -1287,7 +1285,7 @@ int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs)
12871285
return 0;
12881286

12891287
abort:
1290-
esw->mode = MLX5_ESWITCH_NONE;
1288+
esw->mode = MLX5_ESWITCH_LEGACY;
12911289

12921290
if (mode == MLX5_ESWITCH_OFFLOADS)
12931291
mlx5_rescan_drivers(esw->dev);
@@ -1312,13 +1310,13 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs)
13121310
if (!mlx5_esw_allowed(esw))
13131311
return 0;
13141312

1315-
toggle_lag = esw->mode == MLX5_ESWITCH_NONE;
1313+
toggle_lag = !mlx5_esw_is_fdb_created(esw);
13161314

13171315
if (toggle_lag)
13181316
mlx5_lag_disable_change(esw->dev);
13191317

13201318
down_write(&esw->mode_lock);
1321-
if (esw->mode == MLX5_ESWITCH_NONE) {
1319+
if (!mlx5_esw_is_fdb_created(esw)) {
13221320
ret = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, num_vfs);
13231321
} else {
13241322
enum mlx5_eswitch_vport_event vport_events;
@@ -1337,56 +1335,79 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs)
13371335
return ret;
13381336
}
13391337

1340-
void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf)
1338+
/* When disabling sriov, free driver level resources. */
1339+
void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf)
13411340
{
1342-
struct devlink *devlink = priv_to_devlink(esw->dev);
1343-
int old_mode;
1344-
1345-
lockdep_assert_held_write(&esw->mode_lock);
1346-
1347-
if (esw->mode == MLX5_ESWITCH_NONE)
1341+
if (!mlx5_esw_allowed(esw))
13481342
return;
13491343

1350-
esw_info(esw->dev, "Disable: mode(%s), nvfs(%d), active vports(%d)\n",
1344+
down_write(&esw->mode_lock);
1345+
/* If driver is unloaded, this function is called twice by remove_one()
1346+
* and mlx5_unload(). Prevent the second call.
1347+
*/
1348+
if (!esw->esw_funcs.num_vfs && !clear_vf)
1349+
goto unlock;
1350+
1351+
esw_info(esw->dev, "Unload vfs: mode(%s), nvfs(%d), active vports(%d)\n",
13511352
esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",
13521353
esw->esw_funcs.num_vfs, esw->enabled_vports);
13531354

1355+
mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
1356+
if (clear_vf)
1357+
mlx5_eswitch_clear_vf_vports_info(esw);
1358+
/* If disabling sriov in switchdev mode, free meta rules here
1359+
* because it depends on num_vfs.
1360+
*/
1361+
if (esw->mode == MLX5_ESWITCH_OFFLOADS) {
1362+
struct devlink *devlink = priv_to_devlink(esw->dev);
1363+
1364+
esw_offloads_del_send_to_vport_meta_rules(esw);
1365+
devlink_rate_nodes_destroy(devlink);
1366+
}
1367+
1368+
esw->esw_funcs.num_vfs = 0;
1369+
1370+
unlock:
1371+
up_write(&esw->mode_lock);
1372+
}
1373+
1374+
/* Free resources for corresponding eswitch mode. It is called by devlink
1375+
* when changing eswitch mode or modprobe when unloading driver.
1376+
*/
1377+
void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw)
1378+
{
1379+
struct devlink *devlink = priv_to_devlink(esw->dev);
1380+
13541381
/* Notify eswitch users that it is exiting from current mode.
13551382
* So that it can do necessary cleanup before the eswitch is disabled.
13561383
*/
1357-
mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_NONE);
1384+
mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_LEGACY);
13581385

13591386
mlx5_eswitch_event_handlers_unregister(esw);
13601387

1388+
esw_info(esw->dev, "Disable: mode(%s), nvfs(%d), active vports(%d)\n",
1389+
esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",
1390+
esw->esw_funcs.num_vfs, esw->enabled_vports);
1391+
13611392
esw->fdb_table.flags &= ~MLX5_ESW_FDB_CREATED;
1362-
if (esw->mode == MLX5_ESWITCH_LEGACY)
1363-
esw_legacy_disable(esw);
1364-
else if (esw->mode == MLX5_ESWITCH_OFFLOADS)
1393+
if (esw->mode == MLX5_ESWITCH_OFFLOADS)
13651394
esw_offloads_disable(esw);
1366-
1367-
old_mode = esw->mode;
1368-
esw->mode = MLX5_ESWITCH_NONE;
1369-
1370-
if (old_mode == MLX5_ESWITCH_OFFLOADS)
1371-
mlx5_rescan_drivers(esw->dev);
1372-
1373-
devlink_rate_nodes_destroy(devlink);
1374-
1395+
else if (esw->mode == MLX5_ESWITCH_LEGACY)
1396+
esw_legacy_disable(esw);
13751397
mlx5_esw_acls_ns_cleanup(esw);
13761398

1377-
if (clear_vf)
1378-
mlx5_eswitch_clear_vf_vports_info(esw);
1399+
if (esw->mode == MLX5_ESWITCH_OFFLOADS)
1400+
devlink_rate_nodes_destroy(devlink);
13791401
}
13801402

1381-
void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf)
1403+
void mlx5_eswitch_disable(struct mlx5_eswitch *esw)
13821404
{
13831405
if (!mlx5_esw_allowed(esw))
13841406
return;
13851407

13861408
mlx5_lag_disable_change(esw->dev);
13871409
down_write(&esw->mode_lock);
1388-
mlx5_eswitch_disable_locked(esw, clear_vf);
1389-
esw->esw_funcs.num_vfs = 0;
1410+
mlx5_eswitch_disable_locked(esw);
13901411
up_write(&esw->mode_lock);
13911412
mlx5_lag_enable_change(esw->dev);
13921413
}
@@ -1581,7 +1602,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
15811602
refcount_set(&esw->qos.refcnt, 0);
15821603

15831604
esw->enabled_vports = 0;
1584-
esw->mode = MLX5_ESWITCH_NONE;
1605+
esw->mode = MLX5_ESWITCH_LEGACY;
15851606
esw->offloads.inline_mode = MLX5_INLINE_MODE_NONE;
15861607
if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) &&
15871608
MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))
@@ -1883,7 +1904,7 @@ u8 mlx5_eswitch_mode(const struct mlx5_core_dev *dev)
18831904
{
18841905
struct mlx5_eswitch *esw = dev->priv.eswitch;
18851906

1886-
return mlx5_esw_allowed(esw) ? esw->mode : MLX5_ESWITCH_NONE;
1907+
return mlx5_esw_allowed(esw) ? esw->mode : MLX5_ESWITCH_LEGACY;
18871908
}
18881909
EXPORT_SYMBOL_GPL(mlx5_eswitch_mode);
18891910

‎drivers/net/ethernet/mellanox/mlx5/core/eswitch.h

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,7 @@ void esw_offloads_disable(struct mlx5_eswitch *esw);
342342
int esw_offloads_enable(struct mlx5_eswitch *esw);
343343
void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw);
344344
int esw_offloads_init_reps(struct mlx5_eswitch *esw);
345+
void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw);
345346

346347
bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw);
347348
int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable);
@@ -357,8 +358,9 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw);
357358
#define MLX5_ESWITCH_IGNORE_NUM_VFS (-1)
358359
int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs);
359360
int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs);
360-
void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf);
361-
void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf);
361+
void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf);
362+
void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw);
363+
void mlx5_eswitch_disable(struct mlx5_eswitch *esw);
362364
int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw,
363365
u16 vport, const u8 *mac);
364366
int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw,
@@ -729,7 +731,8 @@ int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw);
729731
static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; }
730732
static inline void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) {}
731733
static inline int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) { return 0; }
732-
static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf) {}
734+
static inline void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) {}
735+
static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw) {}
733736
static inline bool mlx5_eswitch_is_funcs_handler(struct mlx5_core_dev *dev) { return false; }
734737
static inline
735738
int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, u16 vport, int link_state) { return 0; }

‎drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c

Lines changed: 15 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1040,6 +1040,15 @@ static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw)
10401040
mlx5_del_flow_rules(flows[i]);
10411041

10421042
kvfree(flows);
1043+
/* If changing eswitch mode from switchdev to legacy, but num_vfs is not 0,
1044+
* meta rules could be freed again. So set it to NULL.
1045+
*/
1046+
esw->fdb_table.offloads.send_to_vport_meta_rules = NULL;
1047+
}
1048+
1049+
void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw)
1050+
{
1051+
mlx5_eswitch_del_send_to_vport_meta_rules(esw);
10431052
}
10441053

10451054
static int
@@ -2034,7 +2043,7 @@ static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
20342043
if (!MLX5_CAP_GEN(dev, vport_group_manager))
20352044
return -EOPNOTSUPP;
20362045

2037-
if (esw->mode == MLX5_ESWITCH_NONE)
2046+
if (!mlx5_esw_is_fdb_created(esw))
20382047
return -EOPNOTSUPP;
20392048

20402049
switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
@@ -2170,7 +2179,6 @@ static int esw_offloads_start(struct mlx5_eswitch *esw,
21702179
{
21712180
int err, err1;
21722181

2173-
mlx5_eswitch_disable_locked(esw, false);
21742182
err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS,
21752183
esw->dev->priv.sriov.num_vfs);
21762184
if (err) {
@@ -2894,7 +2902,7 @@ int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable)
28942902
int err = 0;
28952903

28962904
down_write(&esw->mode_lock);
2897-
if (esw->mode != MLX5_ESWITCH_NONE) {
2905+
if (mlx5_esw_is_fdb_created(esw)) {
28982906
err = -EBUSY;
28992907
goto done;
29002908
}
@@ -3229,7 +3237,6 @@ static int esw_offloads_stop(struct mlx5_eswitch *esw,
32293237
{
32303238
int err, err1;
32313239

3232-
mlx5_eswitch_disable_locked(esw, false);
32333240
err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY,
32343241
MLX5_ESWITCH_IGNORE_NUM_VFS);
32353242
if (err) {
@@ -3334,15 +3341,6 @@ static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
33343341
return 0;
33353342
}
33363343

3337-
static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw)
3338-
{
3339-
/* devlink commands in NONE eswitch mode are currently supported only
3340-
* on ECPF.
3341-
*/
3342-
return (esw->mode == MLX5_ESWITCH_NONE &&
3343-
!mlx5_core_is_ecpf_esw_manager(esw->dev)) ? -EOPNOTSUPP : 0;
3344-
}
3345-
33463344
/* FIXME: devl_unlock() followed by devl_lock() inside driver callback
33473345
* is never correct and prone to races. It's a transitional workaround,
33483346
* never repeat this pattern.
@@ -3399,6 +3397,7 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
33993397
if (cur_mlx5_mode == mlx5_mode)
34003398
goto unlock;
34013399

3400+
mlx5_eswitch_disable_locked(esw);
34023401
if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) {
34033402
if (mlx5_devlink_trap_get_num_active(esw->dev)) {
34043403
NL_SET_ERR_MSG_MOD(extack,
@@ -3409,6 +3408,7 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
34093408
err = esw_offloads_start(esw, extack);
34103409
} else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) {
34113410
err = esw_offloads_stop(esw, extack);
3411+
mlx5_rescan_drivers(esw->dev);
34123412
} else {
34133413
err = -EINVAL;
34143414
}
@@ -3431,12 +3431,7 @@ int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
34313431
return PTR_ERR(esw);
34323432

34333433
mlx5_eswtich_mode_callback_enter(devlink, esw);
3434-
err = eswitch_devlink_esw_mode_check(esw);
3435-
if (err)
3436-
goto unlock;
3437-
34383434
err = esw_mode_to_devlink(esw->mode, mode);
3439-
unlock:
34403435
mlx5_eswtich_mode_callback_exit(devlink, esw);
34413436
return err;
34423437
}
@@ -3485,9 +3480,6 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
34853480
return PTR_ERR(esw);
34863481

34873482
mlx5_eswtich_mode_callback_enter(devlink, esw);
3488-
err = eswitch_devlink_esw_mode_check(esw);
3489-
if (err)
3490-
goto out;
34913483

34923484
switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
34933485
case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
@@ -3539,12 +3531,7 @@ int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
35393531
return PTR_ERR(esw);
35403532

35413533
mlx5_eswtich_mode_callback_enter(devlink, esw);
3542-
err = eswitch_devlink_esw_mode_check(esw);
3543-
if (err)
3544-
goto unlock;
3545-
35463534
err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode);
3547-
unlock:
35483535
mlx5_eswtich_mode_callback_exit(devlink, esw);
35493536
return err;
35503537
}
@@ -3555,16 +3542,13 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
35553542
{
35563543
struct mlx5_core_dev *dev = devlink_priv(devlink);
35573544
struct mlx5_eswitch *esw;
3558-
int err;
3545+
int err = 0;
35593546

35603547
esw = mlx5_devlink_eswitch_get(devlink);
35613548
if (IS_ERR(esw))
35623549
return PTR_ERR(esw);
35633550

35643551
mlx5_eswtich_mode_callback_enter(devlink, esw);
3565-
err = eswitch_devlink_esw_mode_check(esw);
3566-
if (err)
3567-
goto unlock;
35683552

35693553
if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
35703554
(!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
@@ -3615,21 +3599,15 @@ int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
36153599
enum devlink_eswitch_encap_mode *encap)
36163600
{
36173601
struct mlx5_eswitch *esw;
3618-
int err;
36193602

36203603
esw = mlx5_devlink_eswitch_get(devlink);
36213604
if (IS_ERR(esw))
36223605
return PTR_ERR(esw);
36233606

36243607
mlx5_eswtich_mode_callback_enter(devlink, esw);
3625-
err = eswitch_devlink_esw_mode_check(esw);
3626-
if (err)
3627-
goto unlock;
3628-
36293608
*encap = esw->offloads.encap;
3630-
unlock:
36313609
mlx5_eswtich_mode_callback_exit(devlink, esw);
3632-
return err;
3610+
return 0;
36333611
}
36343612

36353613
static bool

‎drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -632,6 +632,7 @@ static int mlx5_deactivate_lag(struct mlx5_lag *ldev)
632632
static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev)
633633
{
634634
#ifdef CONFIG_MLX5_ESWITCH
635+
struct mlx5_core_dev *dev;
635636
u8 mode;
636637
#endif
637638
int i;
@@ -641,11 +642,11 @@ static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev)
641642
return false;
642643

643644
#ifdef CONFIG_MLX5_ESWITCH
644-
mode = mlx5_eswitch_mode(ldev->pf[MLX5_LAG_P1].dev);
645-
646-
if (mode != MLX5_ESWITCH_NONE && mode != MLX5_ESWITCH_OFFLOADS)
645+
dev = ldev->pf[MLX5_LAG_P1].dev;
646+
if ((mlx5_sriov_is_enabled(dev)) && !is_mdev_switchdev_mode(dev))
647647
return false;
648648

649+
mode = mlx5_eswitch_mode(dev);
649650
for (i = 0; i < ldev->ports; i++)
650651
if (mlx5_eswitch_mode(ldev->pf[i].dev) != mode)
651652
return false;
@@ -760,8 +761,7 @@ static bool mlx5_lag_is_roce_lag(struct mlx5_lag *ldev)
760761

761762
#ifdef CONFIG_MLX5_ESWITCH
762763
for (i = 0; i < ldev->ports; i++)
763-
roce_lag = roce_lag &&
764-
ldev->pf[i].dev->priv.eswitch->mode == MLX5_ESWITCH_NONE;
764+
roce_lag = roce_lag && is_mdev_legacy_mode(ldev->pf[i].dev);
765765
#endif
766766

767767
return roce_lag;

‎drivers/net/ethernet/mellanox/mlx5/core/main.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1250,6 +1250,7 @@ static void mlx5_unload(struct mlx5_core_dev *dev)
12501250
{
12511251
mlx5_sf_dev_table_destroy(dev);
12521252
mlx5_sriov_detach(dev);
1253+
mlx5_eswitch_disable(dev->priv.eswitch);
12531254
mlx5_lag_remove_mdev(dev);
12541255
mlx5_ec_cleanup(dev);
12551256
mlx5_sf_hw_table_destroy(dev);

‎drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -501,7 +501,7 @@ static int mlx5_sf_esw_event(struct notifier_block *nb, unsigned long event, voi
501501
case MLX5_ESWITCH_OFFLOADS:
502502
mlx5_sf_table_enable(table);
503503
break;
504-
case MLX5_ESWITCH_NONE:
504+
case MLX5_ESWITCH_LEGACY:
505505
mlx5_sf_table_disable(table);
506506
break;
507507
default:

0 commit comments

Comments
 (0)