Mellanox ConnectX-4 Lx EN network interface card; 25GbE single-port SFP28; PCIe3.0 x8; ROHS R6 (Installed in PCI3 x16 x4 slot /w SFP28 100ft OM3 fiber).
Same system, no changes to BIOS, no changes to hardware. Different boot selection during power-on sequence:
- Windows 10 Pro 22H2 : Card is always recognized by the system but device manager reports that it has failed to initialize (Code 10). Workaround is to reboot until success.
- Ubuntu 24.10 Flashboot : starts consistently. With mlx5_core driver the card is perfectly functional.
Ubuntu operates at full wire speed. Windows slightly more than half wire speed. There are a pile of card configuration parameters. Soft reset using mlxfwreset does not clear the problem. Currently monitoring mlxconfig SW_RECOVERY_ON_ERRORS=1 configuration setting.
I am drowning in configuration options that look like they could help if only I knew what they do. Documentation for ConnectX-4 Lx driver settings would help a lot.
Previously, connectX3 drivers failed because of Windows 10 e-core incompatibility. Rescued the situation by changing to Intel. History may repeat.
I’m prepared to change motherboards or buy Intel E810-XXVDA2 or Intel Ethernet XXV710-DA2. Recommendations or ideas welcome.
FLEX_PARSER_PROFILE_ENABLE = 0
FLEX_IPV4_OVER_VXLAN_PORT = 0
SWITCH_COMPT_FEATURE_MASK = 0x0(0)
PF_NUM_OF_VF_VALID = False(0)
NON_PREFETCHABLE_PF_BAR = False(0)
VF_VPD_ENABLE = False(0)
STRICT_VF_MSIX_NUM = False(0)
VF_NODNIC_ENABLE = False(0)
NUM_PF_MSIX_VALID = True(1)
NUM_OF_VFS = 8
NUM_OF_PF = 1
SRIOV_EN = True(1)
PF_LOG_BAR_SIZE = 5
VF_LOG_V_SIZE = 0
NUM_PF_MSIX = 63
NUM_VF_MSIX = 11
PCIE_CREDIT_TOKEN_TIMEOUT = 0
PCI_BUS0_RESTRICT_SPEED = PCI_GEN_1(0)
PCI_BUS0_RESTRICT_ASPM = False(0)
PCI_BUS0_RESTRICT_WIDTH = PCI_X1(0)
PCI_BUS0_RESTRICT = False(0)
PCI_DOWNSTREAM_PORT_OWNER = Array[0..15]
LOG_DCR_HASH_TABLE_SIZE = 14
MAX_PACKET_LIFETIME = 0
DCR_LIFO_SIZE = 16384
MEMIC_BAR_SIZE = 0
MEMIC_SIZE_LIMIT = _256KB(1)
ICM_CACHE_MODE = DEVICE_DEFAULT(0)
TX_SCHEDULER_BURST = 0
ACCURATE_TX_SCHEDULER = False(0)
PARTIAL_RESET_EN = False(0)
SW_RECOVERY_ON_ERRORS = True(1)
RESET_WITH_HOST_ON_ERRORS = False(0)
ROCE_NEXT_PROTOCOL = 254
LOG_MAX_QUEUE = 17
CQE_COMPRESSION = BALANCED(0)
IP_OVER_VXLAN_EN = False(0)
MKEY_BY_NAME = False(0)
UCTX_EN = True(1)
PCI_ATOMIC_MODE = PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)
TUNNEL_ECN_COPY_DISABLE= False(0)
LRO_LOG_TIMEOUT0 = 6
LRO_LOG_TIMEOUT1 = 7
LRO_LOG_TIMEOUT2 = 8
LRO_LOG_TIMEOUT3 = 13
INT_LOG_MAX_PAYLOAD_SIZE = AUTOMATIC(0)
IB_PROTO_WIDTH_EN_MASK_P1 = 0
KEEP_ETH_LINK_UP_P1 = True(1)
KEEP_IB_LINK_UP_P1 = False(0)
KEEP_LINK_UP_ON_BOOT_P1 = False(0)
KEEP_LINK_UP_ON_STANDBY_P1 = False(0)
DO_NOT_CLEAR_PORT_STATS_P1 = False(0)
AUTO_POWER_SAVE_LINK_DOWN_P1 = False(0)
PHY_FEC_OVERRIDE_P1 = DEVICE_DEFAULT(0)
LLDP_NB_DCBX_P1 = False(0)
LLDP_NB_RX_MODE_P1 = OFF(0)
LLDP_NB_TX_MODE_P1 = OFF(0)
DCBX_IEEE_P1 = True(1)
DCBX_CEE_P1 = True(1)
DCBX_WILLING_P1 = True(1)
DUP_MAC_ACTION_P1 = LAST_CFG(0)
SRIOV_IB_ROUTING_MODE_P1 = LID(1)
IB_ROUTING_MODE_P1 = LID(1)
NUM_OF_PLANES_P1 = 0
NUM_OF_VL_P1 = _4_VLs(3)
NUM_OF_TC_P1 = _8_TCs(0)
NUM_OF_PFC_P1 = 8
VL15_BUFFER_SIZE_P1 = 0
ROCE_CC_PRIO_MASK_P1 = 255
ROCE_CC_CNP_MODERATION_P1 = DEVICE_DEFAULT(0)
CLAMP_TGT_RATE_AFTER_TIME_INC_P1 = True(1)
CLAMP_TGT_RATE_P1 = False(0)
RPG_TIME_RESET_P1 = 300
RPG_BYTE_RESET_P1 = 32767
RPG_THRESHOLD_P1 = 1
RPG_MAX_RATE_P1 = 0
RPG_AI_RATE_P1 = 5
RPG_HAI_RATE_P1 = 50
RPG_GD_P1 = 11
RPG_MIN_DEC_FAC_P1 = 50
RPG_MIN_RATE_P1 = 1
RATE_TO_SET_ON_FIRST_CNP_P1 = 0
DCE_TCP_G_P1 = 1019
DCE_TCP_RTT_P1 = 1
RATE_REDUCE_MONITOR_PERIOD_P1 = 4
INITIAL_ALPHA_VALUE_P1 = 1023
MIN_TIME_BETWEEN_CNPS_P1 = 4
CNP_802P_PRIO_P1 = 6
CNP_DSCP_P1 = 48
ROCE_RTT_RESP_DSCP_P1 = 0
ROCE_RTT_RESP_DSCP_MODE_P1 = DEVICE_DEFAULT(0)
PORT_OWNER = True(1)
ALLOW_RD_COUNTERS = True(1)
RENEG_ON_CHANGE = True(1)
TRACER_ENABLE = True(1)
ROCE_CONTROL = ROCE_ENABLE(2)
PCI_WR_ORDERING = per_mkey(0)
MULTI_PORT_VHCA_EN = False(0)
PF_SD_GROUP = 0
BOOT_VLAN = 1
LEGACY_BOOT_PROTOCOL = PXE(1)
BOOT_INTERRUPT_DIS = False(0)
BOOT_LACP_DIS = True(1)
BOOT_VLAN_EN = False(0)
IP_VER = IPv4(0)
BOOT_UNDI_NETWORK_WAIT = 0
BOOT_DBG_LOG = False(0)
BOOT_PKEY = 0
UEFI_HII_EN = True(1)
UEFI_LOGS = DISABLED(0)
DYNAMIC_VF_MSIX_TABLE = False(0)
EXP_ROM_UEFI_ARM_ENABLE = False(0)
EXP_ROM_UEFI_x86_ENABLE = False(0)
EXP_ROM_PXE_ENABLE = True(1)
ADVANCED_PCI_SETTINGS = False(0)
SAFE_MODE_THRESHOLD = 10
SAFE_MODE_ENABLE = True(1)