[Open vSwitch] The function calls about the command: "ovs-vsctl add-port"

  Well...it has been a busy month for me so that I don't have too much extra time do something interesting. But, previously I studied a little bit about the source code of Open vSwitcch and drawed some diagrams about its functionality. Here is the one example for the function calls about the command: "ovs-vsctl add-port". Basically the function call is from user space to kernel with Netlink. It needs a lot of time to explain them. So I only paste the diagram for the reference.

[Xorp] How to use Xorp in RouteFlow

If you use RouteFlow before, you will notice that the routing software it installs is Quagga by default. So, how to change to Xorp in RouteFlow? Here is an example for rftest2 case to do this. I assume the current path is under RouteFlow's directory.

1. Add new configuration files in rftest/config for rfvmA to rfvmD:

rftest/config/rfvmA/rootfs/usr/etc/xorp.conf
    protocols {
        ospf4 {
            router-id: 1.1.1.1
            area 0.0.0.0 {
                interface eth2 {
                    vif eth2 {
                        address 10.0.0.1 {
                        }
                    }
                }
            }
            area 0.0.0.1 {
                interface eth3 {
                    vif eth3 {
                        address 30.0.0.1 {
                        }
                    }
                }
            }
            area 0.0.0.4 {
                interface eth4 {
                    vif eth4 {
                        address 50.0.0.1 {
                        }
                    }
                }
            }
            export: "redis.connect"
        }
    }
    policy {
        policy-statement "redis.connect" {
            term connect {
                from {
                    protocol: "connected"
                }
            }
        }
    }
    fea {
        unicast-forwarding4 {
        }
    }
    interfaces {
    interface eth1 {
            description: "eth1"
            vif eth1 {
                address 172.31.1.1 {
                    prefix-length: 24
                }
            }
        }
        interface eth2 {
            description: "eth2"
            vif eth2 {
                address 10.0.0.1 {
                    prefix-length: 24
                }
            }
        }
        interface eth3 {
            description: "eth3"
            vif eth3 {
                address 30.0.0.1 {
                    prefix-length: 24
                }
            }
        }
        interface eth4 {
            description: "eth4"
            vif eth4 {
                address 50.0.0.1 {
                    prefix-length: 24
                }
            }
        }
        interface lo {
            vif lo {
                address 1.1.1.1 {
                    prefix-length: 32
                }
            }
        }
    }

rftest/config/rfvmB/rootfs/usr/etc/xorp.conf
    protocols {
        ospf4 {
            router-id: 2.2.2.2
            area 0.0.0.0 {
                interface eth2 {
                    vif eth2 {
                        address 10.0.0.2 {
                        }
                    }
                }
            }
            area 0.0.0.2 {
                interface eth3 {
                    vif eth3 {
                        address 40.0.0.2 {
                        }
                    }
                }
            }
            export: "redis.connect"
        }
    }
    policy {
        policy-statement "redis.connect" {
            term connect {
                from {
                    protocol: "connected"
                }
            }
        }
    }
    fea {
        unicast-forwarding4 {
        }
    }
    interfaces {
    interface eth1 {
            description: "eth1"
            vif eth1 {
                address 172.31.2.1 {
                    prefix-length: 24
                }
            }
        }
        interface eth2 {
            description: "eth2"
            vif eth2 {
                address 10.0.0.2 {
                    prefix-length: 24
                }
            }
        }
        interface eth3 {
            description: "eth3"
            vif eth3 {
                address 40.0.0.2 {
                    prefix-length: 24
                }
            }
        }
        interface lo {
            vif lo {
                address 2.2.2.2 {
                    prefix-length: 32
                }
            }
        }

    }

rftest/config/rfvmC/rootfs/usr/etc/xorp.conf
    protocols {
        ospf4 {
            router-id: 3.3.3.3
            area 0.0.0.1 {
                interface eth3 {
                    vif eth3 {
                        address 30.0.0.3 {
                        }
                    }
                }
            }
            area 0.0.0.3 {
                interface eth2 {
                    vif eth2 {
                        address 20.0.0.3 {
                        }
                    }
                }
            }

            export: "redis.connect"
        }
    }
    policy {
        policy-statement "redis.connect" {
            term connect {
                from {
                    protocol: "connected"
                }
            }
        }
    }
    fea {
        unicast-forwarding4 {
        }
    }
    interfaces {
    interface eth1 {
            description: "eth1"
            vif eth1 {
                address 172.31.3.1 {
                    prefix-length: 24
                }
            }
        }
        interface eth2 {
            description: "eth2"
            vif eth2 {
                address 20.0.0.3 {
                    prefix-length: 24
                }
            }
        }
        interface eth3 {
            description: "eth3"
            vif eth3 {
                address 30.0.0.3 {
                    prefix-length: 24
                }
            }
        }
        interface lo {
            vif lo {
                address 3.3.3.3 {
                    prefix-length: 32
                }
            }
        }
    }

rftest/config/rfvmD/rootfs/usr/etc/xorp.conf
    protocols {
        ospf4 {
            router-id: 4.4.4.4
            area 0.0.0.2 {
                interface eth2 {
                    vif eth2 {
                        address 40.0.0.4 {
                        }
                    }
                }
            }
            area 0.0.0.3 {
                interface eth3 {
                    vif eth3 {
                        address 20.0.0.4 {
                        }
                    }
                }
            }
            area 0.0.0.4 {
                interface eth4 {
                    vif eth4 {
                        address 50.0.0.4 {
                        }
                    }
                }
            }
            export: "redis.connect"
        }
    }
    policy {
        policy-statement "redis.connect" {
            term connect {
                from {
                    protocol: "connected"
                }
            }
        }
    }
    fea {
        unicast-forwarding4 {
        }
    }
    interfaces {
    interface eth1 {
            description: "eth1"
            vif eth1 {
                address 172.31.4.1 {
                    prefix-length: 24
                }
            }
        }
        interface eth2 {
            description: "eth2"
            vif eth2 {
                address 40.0.0.4 {
                    prefix-length: 24
                }
            }
        }
        interface eth3 {
            description: "eth3"
            vif eth3 {
                address 20.0.0.4 {
                    prefix-length: 24
                }
            }
        }
        interface eth4 {
            description: "eth4"
            vif eth4 {
                address 50.0.0.4 {
                    prefix-length: 24
                }
            }
        }
        interface lo {
            vif lo {
                address 4.4.4.4 {
                    prefix-length: 32
                }
            }
        }
    }

2. Modify the "rftest/create" shell script

chroot $LXCDIR/base/rootfs apt-get -y --force-yes install xorp quagga libboost-thread-dev libboost-system-dev libboost-filesystem-dev libboost-program-options-dev rsyslog vlan tcpdump

chroot $LXCDIR/base/rootfs usermod -a -G xorp root
chroot $LXCDIR/rfvmA/rootfs usermod -a -G xorp root
chroot $LXCDIR/rfvmB/rootfs usermod -a -G xorp root
chroot $LXCDIR/rfvmC/rootfs usermod -a -G xorp root
chroot $LXCDIR/rfvmD/rootfs usermod -a -G xorp root

3. Modify the "rftest/rftest2" shell script

SCRIPT_NAME="rftest_xorp"
...
...
echo "#!/bin/sh" > /var/lib/lxc/rfvmA/rootfs/root/run_rfclient.sh
echo "sleep 3" >> /var/lib/lxc/rfvmA/rootfs/root/run_rfclient.sh
if [ "$SCRIPT_NAME" == "rftest_xorp" ]
then
    echo "sudo xorp_rtrmgr" >> /var/lib/lxc/rfvmA/rootfs/root/run_rfclient.sh
else
    echo "/etc/init.d/quagga start" >> /var/lib/lxc/rfvmA/rootfs/root/run_rfclient.sh
fi
echo "/opt/rfclient/rfclient > /var/log/rfclient.log" >> /var/lib/lxc/rfvmA/rootfs/root/run_rfclient.sh

echo "#!/bin/sh" > /var/lib/lxc/rfvmB/rootfs/root/run_rfclient.sh
echo "sleep 3" >> /var/lib/lxc/rfvmB/rootfs/root/run_rfclient.sh
if [ "$SCRIPT_NAME" == "rftest_xorp" ]
then
    echo "sudo xorp_rtrmgr" >> /var/lib/lxc/rfvmB/rootfs/root/run_rfclient.sh
else
    echo "/etc/init.d/quagga start" >> /var/lib/lxc/rfvmB/rootfs/root/run_rfclient.sh
fi
echo "/opt/rfclient/rfclient > /var/log/rfclient.log" >> /var/lib/lxc/rfvmB/rootfs/root/run_rfclient.sh

echo "#!/bin/sh" > /var/lib/lxc/rfvmC/rootfs/root/run_rfclient.sh
echo "sleep 3" >> /var/lib/lxc/rfvmC/rootfs/root/run_rfclient.sh
if [ "$SCRIPT_NAME" == "rftest_xorp" ]
then
    echo "sudo xorp_rtrmgr" >> /var/lib/lxc/rfvmC/rootfs/root/run_rfclient.sh
else
    echo "/etc/init.d/quagga start" >> /var/lib/lxc/rfvmC/rootfs/root/run_rfclient.sh
fi
echo "/opt/rfclient/rfclient > /var/log/rfclient.log" >> /var/lib/lxc/rfvmC/rootfs/root/run_rfclient.sh

echo "#!/bin/sh" > /var/lib/lxc/rfvmD/rootfs/root/run_rfclient.sh
echo "sleep 3" >> /var/lib/lxc/rfvmD/rootfs/root/run_rfclient.sh
if [ "$SCRIPT_NAME" == "rftest_xorp" ]
then
    echo "sudo xorp_rtrmgr" >> /var/lib/lxc/rfvmD/rootfs/root/run_rfclient.sh
else
    echo "/etc/init.d/quagga start" >> /var/lib/lxc/rfvmD/rootfs/root/run_rfclient.sh
fi
echo "/opt/rfclient/rfclient > /var/log/rfclient.log" >> /var/lib/lxc/rfvmD/rootfs/root/run_rfclient.sh

4. Then, you can execute the script file: "create" and "rftest"

5. Verify  ==> for instance, rfvmD

sudo lxc-console -n rfvmD

ubuntu@rfvmD:~$ sudo xorpsh
[sudo] password for ubuntu:
Welcome to XORP on rfvmD
root@rfvmD> show ospf4 neighbor
  Address         Interface             State      ID              Pri  Dead
40.0.0.2         eth2/eth2              Full      2.2.2.2          128    38
20.0.0.3         eth3/eth3              Full      3.3.3.3          128    39
50.0.0.1         eth4/eth4              Full      1.1.1.1          128    39

root@rfvmD> show interfaces
eth2/eth2: Flags:<ENABLED,BROADCAST,MULTICAST> mtu 1500 speed 1 Gbps
        inet 40.0.0.4 subnet 40.0.0.0/24 broadcast 40.0.0.255
        physical index 161
        ether 2:d2:d2:d2:d2:d2
eth3/eth3: Flags:<ENABLED,BROADCAST,MULTICAST> mtu 1500 speed 1 Gbps
        inet 20.0.0.4 subnet 20.0.0.0/24 broadcast 20.0.0.255
        physical index 163
        ether 2:d3:d3:d3:d3:d3
eth4/eth4: Flags:<ENABLED,BROADCAST,MULTICAST> mtu 1500 speed 1 Gbps
        inet 50.0.0.4 subnet 50.0.0.0/24 broadcast 50.0.0.255
        physical index 165
        ether 2:d4:d4:d4:d4:d4
lo/lo: Flags:<ENABLED,LOOPBACK> mtu 16436 speed unknown
        inet 4.4.4.4 subnet 4.4.4.4/32
        physical index 168
root@rfvmD>

[NETCONF] Try Netopeer ( NETCONF tools )

Netopeer is a set of NETCONF tools built on the libnetconf library. 

It has single level server, multiple level server, and cli to use. 
Basically I think if we don't try, we won't truly understand what NETCONF/YANG look like and how it works. During the compilation and setup, I encounter some problems, but I find the following information that save me a lot of time to do trouble shooting.

Here is a URL about how to setup Netopeer server and to use it with cli.
http://seguesoft.com/how-to-set-up-netopeer-server-to-use-with-netconfc

This URL provides a lot of YANG Modules and examples to refer:
http://www.netconfcentral.org/

[TTP] What is Table Type Patterns?

As we know ( if you have to deal with the multiple flow tables of OF 1.1+ ), the multiple flow tables provide the scaling capability but increasing the complex of usage. For software switch either running on hypervisor or on operation system, it can use memory to map to the implementation of such table design. But for hardware switch based on ASIC to handle packet process pipline, the forwarding pipeline and the definition of flow table is specified and fixed. That will generate a big issue: How come application running on SDN controller knows how to deal with these specified limitation. Recently I noticed "TTP", stands for Table Type Patterns, to allow for an OpenFlow controller and OpenFlow switch to agree on a set of functionality to help manage the increased diversity made possible with OpenFlow versions 1.1+. I think that is very useful for the case: OF-DPA.

https://www.sdncentral.com/education/openflow-table-type-patterns-opendaylight-next-release-colin-dixon/2014/08/
This article mentions these capabilities have made OpenFlow a much more interesting tool in two major ways:

1. Better for developers. Simply put, the new features allow developers to do more interesting things: handle IPv6 traffic, provide better multipathing, and separate logical concerns into different tables. Of course, this is all limited to software switches if hardware doesn’t expose these features via OpenFlow, and it can’t do that if OpenFlow provides poor abstractions for hardware.

2. Better mapping onto hardware. The original model of OpenFlow (a single, very flexible table) was actually a poor fit for real network hardware. The new model allows for an OpenFlow table pipeline that can much more closely match the pipelines in real networking ASICs. This allows hardware to both expose more of its capabilities and expose them to controllers (and thus developers) in a way they can efficiently take advantage of.


https://github.com/OpenNetworkingFoundation/TTP_Repository/blob/master/TTP-FAQ.md
This FAQ document give us more information in details:

TTPs will be particularly helpful in simplifying the coding of advanced OpenFlow datapath control (where many flow tables or optional functions are needed). Because TTPs describe the expected controller/switch messaging unambiguously, they will also improve interoperability.

TTPs will also be helpful in testing or benchmarking contexts where participants want advance notice to ensure conformance or optimize performance. Such participants expect precise descriptions of what messages will be used during testing or benchmarking.

To legitimately claim support for a TTP, a switch must implement all non-optional functionality described by the TTP. As mentioned above, a TTP may describe some functionality that is optional.


PS: OpenDaylight has support TTP in Helium version
https://wiki.opendaylight.org/view/Table_Type_Patterns:Helium_Release_Notes

[OpenFlow] Install OpenFlow Dissector to Wireshark

It is for OpenFlow 1.1 and higher version.

Get sources (for Debian-based)

git clone git://github.com/CPqD/ofdissector

sudo apt-get install wireshark wireshark-dev

export WIRESHARK=/usr/include/wireshark

Install scons and build openflow library

cd ofdissector/src

sudo -s

apt-get install scons

scons install

#mv openflow.so /usr/lib/wireshark/libwireshark1/plugins/

#cd /usr/lib/wireshark/libwireshark1/plugins/

#chown root:root openflow.so

#chmod 644 openflow.so

Run Wireshark.

wireshark

[Linux Kernel] How to build Linux kernel on Ubuntu?

The general way to build Linux kernel:

• apt-get install libncurses-dev kernel-package bzip2 make ncurses-dev fakeroot module-init-tools patch 
• apt-get install fakeroot build-essential kernel-package libncurses5 libncurses5-dev
• To find the stable version from https://www.kernel.org/ and download it and tar the tar.bz2 file into /usr/src/
• cd /usr/src/linux-source-3.2.0/   <== it is my case
• make mrproper
• make menuconfig
• make -j 16
• sudo make modules_install install
• sudo update-grub2
• reboot

Booting Menu:

/boot/grub/grub.cfg

Booting Linux Options:

/etc/default/grub


The Ubuntu way to do so:

• make-kpkg clean
• make-kpkg --initrd --append-to-version=danny4400 kernel_image kernel-headers 
• cd ..
• dpkg -i   linux-image-（Push tab button)
• ls -l /boot/
• reboot

[Lagopus] Install Lagopus software switch on Ubuntu 12.04

 I attended NTT ( Ryu/Lagopus ) seminar in Aug. at NCTU, Taiwan and noticed that Lagopus(SDN/OpenFlow Software Switch) is amazing.

Its L2 switch performance with 10GbE X 2 (RFC2889 test ) for most of packet size are near to 10Gbps and the test platform is 

Intel xeon E5-2660 (8 cores, 16 threads), Intel X520-DA2 DDR3-1600 64GB. 

For more information please see the attachment pictures that I took from the seminar.

The following is the features:

• Best OpenFlow 1.3 compliant software-based switch
• Multi tables, Group tables support
• MPLS, PBB, QinQ, support
• ONF standard specification support
• OpenFlow Switch Specification 1.3.3
• OF-CONFIG 1.1
• Multiple data-plane configuration
• High performance software data-plane on Intel x86 bare-metal server
• Intel DPDK, Raw socket
• Bare metal switch
• Various management/configuration interfaces
• OF-CONFIG, OVSDB, CLI
• SNMP, Ethernet-OAM functionality

For installing Lagopus switch, you can refer to the following URL. It can give us a common installation guide for Lagopus switch.

https://github.com/lagopus/lagopus/blob/master/QUICKSTART.md


About my lagopus environment:sudo vi /usr/local/etc/lagopus/lagopus.conf

 interface {  
   ethernet {  
     eth0;  
     eth1;  
     eth2;  
   }  
 }  
 bridge-domains {  
   br0 {  
     port {  
       eth0;  
       eth1;  
       eth2;  
     }  
     controller {  
       127.0.0.1;  
     }  
   }  
 }  




But, I put 2 shell scripts for quickly installing and setting up the DPDK. I use DPDK-1.6.0 so that all the scripts are based on this version.

compile_dpdk.sh

 #!/bin/sh  
 export RTE_SDK=/home/myname/git/DPDK-1.6.0  
 export RTE_TARGET="x86_64-default-linuxapp-gcc"  
 make config T=${RTE_TARGET}  
 make install T=${RTE_TARGET}  

install_dpdk.sh

 #!/bin/sh  
 export RTE_SDK=/home/myname/git/DPDK-1.6.0
 export RTE_TARGET="x86_64-default-linuxapp-gcc"  
 DPDK_NIC_PCIS="0000:00:08.0 0000:00:09.0 0000:00:0a.0"  
 HUGEPAGE_NOPAGES="1024"  
 set_numa_pages()  
 {  
     for d in /sys/devices/system/node/node? ; do  
         sudo sh -c "echo ${HUGEPAGE_NOPAGES} > $d/hugepages/hugepages-2048kB/nr_hugepages"  
     done  
 }  
 set_no_numa_pages()  
 {  
     sudo sh -c "echo ${HUGEPAGE_NOPAGES} > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"  
 }  
 # install module  
 sudo modprobe uio  
 sudo insmod ${RTE_SDK}/${RTE_TARGET}/kmod/igb_uio.ko  
 sudo insmod ${RTE_SDK}/${RTE_TARGET}/kmod/rte_kni.ko  
 # unbind e1000 NICs from igb and bind igb_uio for DPDK  
 sudo ${RTE_SDK}/tools/pci_unbind.py --bind=igb_uio ${DPDK_NIC_PCIS}  
 sudo ${RTE_SDK}/tools/pci_unbind.py --status  
 # mount fugepagefs  
 echo "Set hugepagesize=${HUGEPAGE_NOPAGES} of 2MB page"  
 NCPUS=$(find /sys/devices/system/node/node? -maxdepth 0 -type d | wc -l)  
 if [ ${NCPUS} -gt 1 ] ; then  
     set_numa_pages  
 else  
     set_no_numa_pages  
 fi  
 echo "Creating /mnt/huge and mounting as hugetlbfs"  
 sudo mkdir -p /mnt/huge  
 grep -s '/mnt/huge' /proc/mounts > /dev/null  
 if [ $? -ne 0 ] ; then  
     sudo mount -t hugetlbfs nodev /mnt/huge  
 fi  
 unset RTE_SDK  
 unset RTE_TARGET  

Here is one thing needs to be notice. 
The variable DPDK_NIC_PCIS is my Linux eth1, eth2, and eth3's bus info as follows:

DPDK_NIC_PCIS="0000:00:08.0 0000:00:09.0 0000:00:0a.0"

You have to change them by running ethtool to see your eth bus info.

So, we need to use the comand "ethtool" to find out the NIC's bus-info as follows:
# ethtool -i eth4
driver: igb
version: 5.0.5-k
firmware-version: 3.11, 0x8000046e
bus-info: 0000:02:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: no


After executing the 2 shell scripts, then we can start the lagopus switch by this:
sudo lagopus -d -- -c3 -n1 -- -p3

[Indigo] The architecture of Indigo 2.0

After checking with the source code of Indigo 2.0, OF-DPA (CDP) ,and IVS on GitHub, I just draw a simple architecture diagram to show the idea of hardware abstraction layer (HAL). I think the most important part is that Big Switches uses this HAL concept as hardware agnostic to adopt the different forwarding engine / port management implementation in different hardware or platform.