Update Proxmox/Archive/ThunderboltNetwork.md
This commit is contained in:
311
Proxmox/Archive/ThunderboltNetwork.md
Normal file
311
Proxmox/Archive/ThunderboltNetwork.md
Normal file
@@ -0,0 +1,311 @@
|
||||
# How to get Thunderbolt Network up and running
|
||||
|
||||
Main idea is from [here](https://gist.github.com/scyto/76e94832927a89d977ea989da157e9dc), but I like to be able to copy paste a bit more comfortable.
|
||||
|
||||
## Assumptions
|
||||
|
||||
This manual was used on Intel NUC 13th generation with 2 TB4 Ports.
|
||||
|
||||
## On all Nodes
|
||||
|
||||
Optional package to track which node can see which other one.
|
||||
```python
|
||||
apt install -y lldpd
|
||||
```
|
||||
|
||||
Mandatory packages.
|
||||
```python
|
||||
apt install -y lsb-release
|
||||
curl -s https://deb.frrouting.org/frr/keys.gpg | sudo tee /usr/share/keyrings/frrouting.gpg > /dev/null
|
||||
FRRVER="frr-stable"
|
||||
echo deb '[signed-by=/usr/share/keyrings/frrouting.gpg]' https://deb.frrouting.org/frr \
|
||||
$(lsb_release -s -c) $FRRVER | sudo tee -a /etc/apt/sources.list.d/frr.list
|
||||
apt update
|
||||
apt install -y frr
|
||||
```
|
||||
|
||||
Add kernel modules.
|
||||
```python
|
||||
# remove empty lines
|
||||
sed '/^$/d' /etc/modules > temp.txt && mv temp.txt /etc/modules
|
||||
|
||||
# add modules
|
||||
tee -a /etc/modules <<EOF
|
||||
thunderbolt
|
||||
thunderbolt-net
|
||||
EOF
|
||||
```
|
||||
|
||||
Add the thunderbolt links.
|
||||
```python
|
||||
tee -a /etc/systemd/network/00-thunderbolt0.link <<EOF
|
||||
[Match]
|
||||
Path=pci-0000:00:0d.2
|
||||
Driver=thunderbolt-net
|
||||
[Link]
|
||||
MACAddressPolicy=none
|
||||
Name=en05
|
||||
EOF
|
||||
|
||||
tee -a /etc/systemd/network/00-thunderbolt1.link <<EOF
|
||||
[Match]
|
||||
Path=pci-0000:00:0d.3
|
||||
Driver=thunderbolt-net
|
||||
[Link]
|
||||
MACAddressPolicy=none
|
||||
Name=en06
|
||||
EOF
|
||||
```
|
||||
|
||||
Automatic setup of interface to be up after reboot or cable insertion.
|
||||
```python
|
||||
tee -a /etc/udev/rules.d/10-tb-en.rules <<EOF
|
||||
ACTION=="move", SUBSYSTEM=="net", KERNEL=="en05", RUN+="/usr/local/bin/pve-en05.sh"
|
||||
ACTION=="move", SUBSYSTEM=="net", KERNEL=="en06", RUN+="/usr/local/bin/pve-en06.sh"
|
||||
EOF
|
||||
|
||||
tee -a /usr/local/bin/pve-en05.sh <<EOF
|
||||
#!/bin/bash
|
||||
/usr/sbin/ifup en05
|
||||
EOF
|
||||
|
||||
tee -a /usr/local/bin/pve-en06.sh <<EOF
|
||||
#!/bin/bash
|
||||
/usr/sbin/ifup en06
|
||||
EOF
|
||||
|
||||
chmod +x /usr/local/bin/pve-en05.sh
|
||||
chmod +x /usr/local/bin/pve-en06.sh
|
||||
```
|
||||
|
||||
Enable IPv4 forwarding.
|
||||
```python3
|
||||
sed -i "s/\#net.ipv4.ip_forward\=1/net.ipv4.ip_forward\=1/" /etc/sysctl.conf
|
||||
```
|
||||
|
||||
Presetup and configuration of FRR.
|
||||
```python
|
||||
sed -i "s/fabricd=no/fabricd=yes/" /etc/frr/daemons
|
||||
systemctl restart frr
|
||||
```
|
||||
|
||||
Make sure interface is coming up!
|
||||
Idea coming from here => https://gist.github.com/scyto/67fdc9a517faefa68f730f82d7fa3570?permalink_comment_id=5077802#gistcomment-5077802
|
||||
```python
|
||||
cat << 'EOF' > /usr/local/bin/restart-frr.sh
|
||||
#!/bin/sh
|
||||
# Delayed start script to tell frr to reload ensuring that it sees thunderbolt links towards other nodes.
|
||||
# condition: is there any tbt network interface and frr service up
|
||||
COUNTER=0
|
||||
while [ ${COUNTER} -lt 5 ]; do
|
||||
sleep 1;
|
||||
TEST=$(ip a | grep ": en0" | grep "UP" | awk 'BEGIN { ORS=""}; {print $2}')
|
||||
if [ ${#TEST} -ge 2 ]; then
|
||||
TEST_SVC=$(service frr status | grep "active (running)")
|
||||
if [ ${#TEST_SVC} -ge 2 ]; then
|
||||
service frr reload;
|
||||
echo "frr service reload request sent"
|
||||
exit 0;
|
||||
fi
|
||||
fi
|
||||
COUNTER=$((COUNTER+1));
|
||||
done
|
||||
echo "Failed to request frr service reload: request NOT sent"
|
||||
exit 1;
|
||||
EOF
|
||||
chmod +x /usr/local/bin/restart-frr.sh
|
||||
|
||||
# create systemd service and make it autoboot
|
||||
tee -a /etc/systemd/system/frr-restarter.service <<EOF
|
||||
[Unit]
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/local/bin/restart-frr.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable frr-restarter
|
||||
```
|
||||
|
||||
## Different settings per Node!
|
||||
|
||||
Adjust the `/etc/network/interfaces`. Remove any section that belongs to any auto added **thunderbolt0** or **thunderbolt1** interface.
|
||||
|
||||
**Node 1**
|
||||
```python
|
||||
sed -i '/iface lo inet loopback/a\
|
||||
\
|
||||
auto lo:0\niface lo:0 inet static\n address 10.0.0.81/32' /etc/network/interfaces
|
||||
sed -i '/^source \/etc\/network\/interfaces\.d\/\*$/d' /etc/network/interfaces
|
||||
sed '${/^$/d;}' /etc/network/interfaces > temp.txt && mv temp.txt /etc/network/interfaces
|
||||
|
||||
tee -a /etc/network/interfaces <<EOF
|
||||
auto en05
|
||||
allow-hotplug en05
|
||||
iface en05 inet manual
|
||||
mtu 65520
|
||||
|
||||
auto en06
|
||||
allow-hotplug en06
|
||||
iface en06 inet manual
|
||||
mtu 65520
|
||||
EOF
|
||||
```
|
||||
|
||||
**Node 2**
|
||||
```python
|
||||
sed -i '/iface lo inet loopback/a\
|
||||
\
|
||||
auto lo:0\niface lo:0 inet static\n address 10.0.0.82/32' /etc/network/interfaces
|
||||
sed -i '/^source \/etc\/network\/interfaces\.d\/\*$/d' /etc/network/interfaces
|
||||
sed '${/^$/d;}' /etc/network/interfaces > temp.txt && mv temp.txt /etc/network/interfaces
|
||||
|
||||
tee -a /etc/network/interfaces <<EOF
|
||||
allow-hotplug en05
|
||||
iface en05 inet manual
|
||||
mtu 65520
|
||||
|
||||
allow-hotplug en06
|
||||
iface en06 inet manual
|
||||
mtu 65520
|
||||
EOF
|
||||
```
|
||||
|
||||
**Node 3**
|
||||
```python
|
||||
sed -i '/iface lo inet loopback/a\
|
||||
\
|
||||
auto lo:0\niface lo:0 inet static\n address 10.0.0.83/32' /etc/network/interfaces
|
||||
sed -i '/^source \/etc\/network\/interfaces\.d\/\*$/d' /etc/network/interfaces
|
||||
sed '${/^$/d;}' /etc/network/interfaces > temp.txt && mv temp.txt /etc/network/interfaces
|
||||
|
||||
tee -a /etc/network/interfaces <<EOF
|
||||
allow-hotplug en05
|
||||
iface en05 inet manual
|
||||
mtu 65520
|
||||
|
||||
allow-hotplug en06
|
||||
iface en06 inet manual
|
||||
mtu 65520
|
||||
EOF
|
||||
```
|
||||
|
||||
Open VTYSH CLI.
|
||||
```python
|
||||
vtysh
|
||||
```
|
||||
|
||||
Enter config mode.
|
||||
```python
|
||||
configure
|
||||
```
|
||||
|
||||
**Node 1**
|
||||
```python
|
||||
ip forwarding
|
||||
!
|
||||
interface en05
|
||||
ip router openfabric 1
|
||||
exit
|
||||
!
|
||||
interface en06
|
||||
ip router openfabric 1
|
||||
exit
|
||||
!
|
||||
interface lo
|
||||
ip router openfabric 1
|
||||
openfabric passive
|
||||
exit
|
||||
!
|
||||
router openfabric 1
|
||||
net 49.0000.0000.0001.00
|
||||
exit
|
||||
!
|
||||
|
||||
end
|
||||
write memory
|
||||
exit
|
||||
|
||||
# Doublecheck correct config
|
||||
vtysh -c "show running-config"
|
||||
```
|
||||
|
||||
**Node 2**
|
||||
```python
|
||||
ip forwarding
|
||||
!
|
||||
interface en05
|
||||
ip router openfabric 1
|
||||
exit
|
||||
!
|
||||
interface en06
|
||||
ip router openfabric 1
|
||||
exit
|
||||
!
|
||||
interface lo
|
||||
ip router openfabric 1
|
||||
openfabric passive
|
||||
exit
|
||||
!
|
||||
router openfabric 1
|
||||
net 49.0000.0000.0002.00
|
||||
exit
|
||||
!
|
||||
|
||||
end
|
||||
write memory
|
||||
exit
|
||||
|
||||
# Doublecheck correct config
|
||||
vtysh -c "show running-config"
|
||||
```
|
||||
|
||||
**Node 3**
|
||||
```python
|
||||
ip forwarding
|
||||
!
|
||||
interface en05
|
||||
ip router openfabric 1
|
||||
exit
|
||||
!
|
||||
interface en06
|
||||
ip router openfabric 1
|
||||
exit
|
||||
!
|
||||
interface lo
|
||||
ip router openfabric 1
|
||||
openfabric passive
|
||||
exit
|
||||
!
|
||||
router openfabric 1
|
||||
net 49.0000.0000.0003.00
|
||||
exit
|
||||
!
|
||||
|
||||
end
|
||||
write memory
|
||||
exit
|
||||
|
||||
# Doublecheck correct config
|
||||
vtysh -c "show running-config"
|
||||
```
|
||||
|
||||
Time for the reboot.
|
||||
```
|
||||
/sbin/reboot
|
||||
```
|
||||
|
||||
## Debugging
|
||||
```python
|
||||
# shows the actual configuration
|
||||
vtysh -c "show running-config"
|
||||
```
|
||||
|
||||
```python
|
||||
# shows all links
|
||||
vtysh -c "show openfabric topology"
|
||||
```
|
||||
Reference in New Issue
Block a user