f-stack安装运行

mac os virtual box跑f-stack环境搭建

基本环境

kernel版本3.10.0-514.el7.x86_64
CentOS-7-x86_64-Minimal-1611.iso
Mac osx
Oreacle Virtual Box5.1.22
kernel-devel

操作步骤

在Oreacle Virtual Box上安装centos 7虚拟机

配置开启NAT网卡和桥接网卡

  • 网卡配置芯片类型选择
    Add two more virtual network adapters with “Intel PRO/1000 MT Server (82545EM)” type in order to provide virtual network hardware to the virtual machine that is supported by Intel DPDK.
    上述说明来自于一篇国外文档说明,使用的Inetl 82545EM,但是经过尝试Intel PRO/1000MT桌面(82540EN)也是可以的。所以不要太迷信啦。
  • 桥接网卡用来ssh登陆管理串口,NAT网卡用来运行DPDK驱动,跑nginx
    在Virtual Box上制作地址映射
    wk1
    wk2
  • centos7虚拟机上网口配置信息
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-enp0s17
TYPE="Ethernet"
BOOTPROTO="dhcp"
DEFROUTE="yes"
PEERDNS="yes"
PEERROUTES="yes"
IPV4_FAILURE_FATAL="no"
IPV6INIT="yes"
IPV6_AUTOCONF="yes"
IPV6_DEFROUTE="yes"
IPV6_PEERDNS="yes"
IPV6_PEERROUTES="yes"
IPV6_FAILURE_FATAL="no"
IPV6_ADDR_GEN_MODE="stable-privacy"
NAME="enp0s17"
UUID="2ea1ed66-7bcd-4153-a495-39c25d5f0ff9"
DEVICE="enp0s17"
ONBOOT="yes"
[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-enp0s8
TYPE=Ethernet
BOOTPROTO=dhcp
DEFROUTE=yes
PEERDNS=yes
PEERROUTES=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_PEERDNS=yes
IPV6_PEERROUTES=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=enp0s8
UUID=6c930d05-bc17-4316-998e-f01a7233cbd3
DEVICE=enp0s8
ONBOOT=yes
[root@localhost ~]# ifconfig
enp0s8: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.55.183 netmask 255.255.255.0 broadcast 192.168.55.255
inet6 fe80::f97d:539:4010:eaff prefixlen 64 scopeid 0x20<link>
ether 08:00:27:d5:ee:00 txqueuelen 1000 (Ethernet)
RX packets 521 bytes 58437 (57.0 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 155 bytes 23680 (23.1 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
enp0s17: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.0.2.15 netmask 255.255.255.0 broadcast 10.0.2.255
inet6 fe80::a15d:3b87:fec0:f3c1 prefixlen 64 scopeid 0x20<link>
ether 08:00:27:28:39:6c txqueuelen 1000 (Ethernet)
RX packets 2 bytes 1180 (1.1 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 10 bytes 1308 (1.2 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

开启CPU flags(SSE 4.1/SSE 4.2)

1
2
VBoxManage setextradata "VM name" VBoxInternal/CPUM/SSE4.1 1
VBoxManage setextradata "VM name" VBoxInternal/CPUM/SSE4.2 1

Note:

  • 上述CPU flags默认是不开启的,启动f-stack上ngx会err日志如下
  • 如果不开启sse cpu选项,在启动ngx会报如下问题
    /usr/local/nginx_fstack/sbin/nginx /data/f-stack/config.ini -c 1 –proc-type=primary –num-procs=1 –proc-id=0
    ERROR: This system does not support “SSE4_1”.
    Please check that RTE_MACHINE is set correctly.
  • 做NAT网卡10.0.2.15需要在vbox上做主机地址和端口映射才能访问
    wk2

安装详细步骤

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
yum install -y git gcc openssl-devel bc
cd /data
yum install kernel-devel-`uname -r` -y
mkdir /data/f-stack
git clone https://github.com/F-Stack/f-stack.git /data/f-stack

# Compile DPDK
cd /data/f-stack/dpdk
make config T=x86_64-native-linuxapp-gcc
make

# set hugepage
echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
mkdir /mnt/huge
mount -t hugetlbfs nodev /mnt/huge

# insmod ko
modprobe uio
insmod build/kmod/igb_uio.ko
insmod build/kmod/rte_kni.ko

# set ip address
export myaddr=`ifconfig enp0s17 | grep "inet" | grep -v ":" | awk -F ' ' '{print $2}'`
export mymask=`ifconfig enp0s17 | grep "netmask" | awk -F ' ' '{print $4}'`
export mybc=`ifconfig enp0s17 | grep "broadcast" | awk -F ' ' '{print $6}'`
export myhw=`ifconfig enp0s17 | grep "ether" | awk -F ' ' '{print $2}'`
export mygw=`route -n | grep 0.0.0.0 | grep enp0s17 | grep UG | awk -F ' ' '{print $2}'`

sed "s/addr=192.168.1.2/addr=${myaddr}/" -i /data/f-stack/config.ini
sed "s/netmask=255.255.255.0/netmask=${mymask}/" -i /data/f-stack/config.ini
sed "s/broadcast=192.168.1.255/broadcast=${mybc}/" -i /data/f-stack/config.ini
sed "s/gateway=192.168.1.1/gateway=${mygw}/" -i /data/f-stack/config.ini


# Compile F-Stack lib
export FF_PATH=/data/f-stack
export FF_DPDK=/data/f-stack/dpdk/build
cd /data/f-stack/lib
make

# Compile Nginx
cd ../app/nginx-1.11.10
./configure --prefix=/usr/local/nginx_fstack --with-ff_module
make
make install

# offload NIC(if there is only one NIC,the follow commands must run in a script)
ifconfig enp0s17 down
python /data/f-stack/dpdk/tools/dpdk-devbind.py --bind=igb_uio enp0s17

# start Nginx
cd ../..
./start.sh -b /usr/local/nginx_fstack/sbin/nginx -c config.ini

测试

在vbox主机上访问映射地址和端口192.168.55.165:8080 -> 10.0.2.15:80
curl http://192.168.55.165:8080

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

附录

f-stack ngx配置文件

/data/f-stack/config.ini

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
[dpdk]
## Hexadecimal bitmask of cores to run on.
lcore_mask=3
## Port mask, enable and disable ports.
## Default: all ports are enabled.
#port_mask=1
channel=4
## Number of ports.
nb_ports=1
promiscuous=1
numa_on=1
## TCP segment offload, default: disabled.
tso=0

## Port config section
## According to dpdk.nb_ports: port0, port1...
[port0]
addr=10.0.2.15
netmask=255.255.255.0
broadcast=10.0.2.255
gateway=10.0.2.2
## Packet capture path, this will hurt performance
#pcap=./a.pcap

## Kni config: if enabled and method=reject,
## all packets that do not belong to the following tcp_port and udp_port
## will transmit to kernel; if method=accept, all packets that belong to
## the following tcp_port and udp_port will transmit to kernel.
#[kni]
#enable=1
#method=reject
#tcp_port=80,443
#udp_port=53

## FreeBSD network performance tuning configurations.
## Most native FreeBSD configurations are supported.
[freebsd.boot]
hz=100

kern.ipc.maxsockets=262144

net.inet.tcp.syncache.hashsize=4096
net.inet.tcp.syncache.bucketlimit=100

net.inet.tcp.tcbhashsize=65536

[freebsd.sysctl]
kern.ipc.somaxconn=32768
kern.ipc.maxsockbuf=16777216

net.inet.tcp.fast_finwait2_recycle=1
net.inet.tcp.sendspace=16384
net.inet.tcp.recvspace=8192
net.inet.tcp.nolocaltimewait=1
net.inet.tcp.cc.algorithm=htcp
net.inet.tcp.sendbuf_max=16777216
net.inet.tcp.recvbuf_max=16777216
net.inet.tcp.sendbuf_auto=1
net.inet.tcp.recvbuf_auto=1
net.inet.tcp.sendbuf_inc=16384
net.inet.tcp.recvbuf_inc=524288
net.inet.tcp.inflight.enable=0
net.inet.tcp.sack=1
net.inet.tcp.blackhole=1
net.inet.tcp.msl=2000
net.inet.tcp.delayed_ack=0

net.inet.udp.blackhole=1
net.inet.ip.redirect=0

f-stack ngx正常启动信息

[root@localhost f-stack]# ./start.sh -b /usr/local/nginx_fstack/sbin/nginx -c config.ini
/usr/local/nginx_fstack/sbin/nginx config.ini -c 1 –proc-type=primary –num-procs=1 –proc-id=0
[dpdk]: lcore_mask=3
[dpdk]: channel=4
[dpdk]: nb_ports=1
[dpdk]: promiscuous=1
[dpdk]: numa_on=1
[dpdk]: tso=0
[port0]: addr=10.0.2.15
[port0]: netmask=255.255.255.0
[port0]: broadcast=10.0.2.255
[port0]: gateway=10.0.2.2
[freebsd.boot]: hz=100
[freebsd.boot]: kern.ipc.maxsockets=262144
[freebsd.boot]: net.inet.tcp.syncache.hashsize=4096
[freebsd.boot]: net.inet.tcp.syncache.bucketlimit=100
[freebsd.boot]: net.inet.tcp.tcbhashsize=65536
[freebsd.sysctl]: kern.ipc.somaxconn=32768
[freebsd.sysctl]: kern.ipc.maxsockbuf=16777216
[freebsd.sysctl]: net.inet.tcp.fast_finwait2_recycle=1
[freebsd.sysctl]: net.inet.tcp.sendspace=16384
[freebsd.sysctl]: net.inet.tcp.recvspace=8192
[freebsd.sysctl]: net.inet.tcp.nolocaltimewait=1
[freebsd.sysctl]: net.inet.tcp.cc.algorithm=htcp
[freebsd.sysctl]: net.inet.tcp.sendbuf_max=16777216
[freebsd.sysctl]: net.inet.tcp.recvbuf_max=16777216
[freebsd.sysctl]: net.inet.tcp.sendbuf_auto=1
[freebsd.sysctl]: net.inet.tcp.recvbuf_auto=1
[freebsd.sysctl]: net.inet.tcp.sendbuf_inc=16384
[freebsd.sysctl]: net.inet.tcp.recvbuf_inc=524288
[freebsd.sysctl]: net.inet.tcp.inflight.enable=0
[freebsd.sysctl]: net.inet.tcp.sack=1
[freebsd.sysctl]: net.inet.tcp.blackhole=1
[freebsd.sysctl]: net.inet.tcp.msl=2000
[freebsd.sysctl]: net.inet.tcp.delayed_ack=0
[freebsd.sysctl]: net.inet.udp.blackhole=1
[freebsd.sysctl]: net.inet.ip.redirect=0
EAL: Detected 1 lcore(s)
EAL: Probing VFIO support…
EAL: PCI device 0000:00:08.0 on NUMA socket -1
EAL: probe driver: 8086:100f rte_em_pmd
EAL: PCI device 0000:00:11.0 on NUMA socket -1
EAL: probe driver: 8086:100f rte_em_pmd
create mbuf pool on socket 0
create ring:arp_ring_0_0 success, 2047 ring entries are now free!
Port 0 MAC: 08 00 27 28 39 6c
TSO is disabled
set port 0 to promiscuous mode ok

Checking link status………………..done
Port 0 Link Up - speed 1000 Mbps - full-duplex
link_elf_lookup_symbol: missing symbol hash table
link_elf_lookup_symbol: missing symbol hash table
netisr_init: forcing maxthreads from 1 to 0
Timecounters tick every 10.000 msec
Timecounter “ff_clock” frequency 100 Hz quality 1
f-stack-0: Ethernet address: 08:00:27:28:39:6c

f-stack 环境安装完成后重启后应该重新设置的参数

下述命令可放入/etc/rd.c/rd.local启动文件中,在机器重启后执行一次下面环境设置

1
2
3
4
5
6
7
8
echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
mount -t hugetlbfs nodev /mnt/huge
modprobe uio
insmod /data/f-stack/dpdk/build/kmod/igb_uio.ko
insmod /data/f-stack/dpdk/build/kmod/rte_kni.ko
ifconfig enp0s17 down
python /data/f-stack/dpdk/tools/dpdk-devbind.py --bind=igb_uio enp0s17
/data/f-stack/start.sh -b /usr/local/nginx_fstack/sbin/nginx -c /data/f-stack/config.ini

如果kernel-devel yum无法找到内核对应版本可去centos官网查找下载

wget https://buildlogs.centos.org/c7.1511.00/kernel/20151119220809/3.10.0-327.el7.x86_64/kernel-devel-3.10.0-327.el7.x86_64.rpm
rpm -ivh kernel-devel-3.10.0-327.el7.x86_64.rpm