diff --git a/breakwater-f-stack/.gitignore b/breakwater-f-stack/.gitignore new file mode 100644 index 0000000..84c048a --- /dev/null +++ b/breakwater-f-stack/.gitignore @@ -0,0 +1 @@ +/build/ diff --git a/breakwater-f-stack/Makefile b/breakwater-f-stack/Makefile new file mode 100644 index 0000000..5881646 --- /dev/null +++ b/breakwater-f-stack/Makefile @@ -0,0 +1,30 @@ +# Copied and modified from https://github.com/F-Stack/f-stack/blob/dev/example/Makefile +SERVER_SOURCES := parser.c framebuffer.c breakwater-f-stack.c + +ifeq ($(FF_PATH),) +$(error "Please export the environment variable `FF_PATH` and point it to the f-stack root directory. This Makefile assumes you have run 'make' in the '/lib' folder") +endif + +ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) +$(error "No installation of DPDK found, maybe you should export environment variable `PKG_CONFIG_PATH`") +endif + +PKGCONF ?= pkg-config + +CFLAGS += -O3 -g -gdwarf-2 $(shell $(PKGCONF) --cflags libdpdk) + +LIBS+= $(shell $(PKGCONF) --static --libs libdpdk) +LIBS+= -I${FF_PATH}/lib -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive +LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread -lnuma +# Start the binary with "sudo LD_LIBRARY_PATH=../target/release build/breakwater-f-stack" +LIBS += -l breakwater_parser_c_bindings -L ../target/release/ + +TARGET="breakwater-f-stack" +all: + mkdir -p build/ + cc ${CFLAGS} -DINET6 -o build/${TARGET} $(SERVER_SOURCES) ${LIBS} + cc ${CFLAGS} -o build/${TARGET}-v4-only $(SERVER_SOURCES) ${LIBS} + +.PHONY: clean +clean: + rm -rf build/ diff --git a/breakwater-f-stack/README.md b/breakwater-f-stack/README.md new file mode 100644 index 0000000..23be096 --- /dev/null +++ b/breakwater-f-stack/README.md @@ -0,0 +1,134 @@ +`apt install dpdk libdpdk-dev make pkg-config libnuma-dev libsystemd-dev ethtool` + +### Build f-stack + +Hint: The following `default.nix` might be helpful + +``` +{ + nixpkgs ? import {}, + nixpkgsUnstable ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/nixos-unstable.tar.gz") {}, +}: + +nixpkgs.mkShell { + buildInputs = [ + nixpkgs.pkg-config + nixpkgs.meson + nixpkgs.ninja + + nixpkgs.dpdk + nixpkgs.openssl # Needed by f-stack + nixpkgs.numactl + + # Needed by the redis example app + nixpkgs.zlib + nixpkgs.jemalloc + nixpkgs.jansson + nixpkgs.libpcap + # nixpkgs.libnfnetlink + nixpkgs.libnl + nixpkgs.libelf + + # nixpkgsUnstable.dpdk # Uncomment to use dpdk from nixpkgs-unstable + # nixpkgs.imagemagick + nixpkgs.python312Packages.pyelftools # needed for dpdk-pmdinfo.py + + # For debugging + # nixpkgs.gdb + ]; +} +``` + +`sbernauer@debian:~/pixelflut/f-stack/lib$ make -j 8` + +`sbernauer@debian:~/pixelflut/f-stack/example$ make # Only needed for testing` + +### Build breakwater-f-stack + +`export FF_PATH=/home/sbernauer/pixelflut/f-stack/` + +You also need to build the `breakwater-parser-c-bindings`, so that the C program can link and use the `breakwater-parser` Rust functions. +You can do that using `cargo build --release -p breakwater-parser-c-bindings` from the git root. + +Afterwards, in this folder, run `make`. + +### Run breakwater-f-stack server + +#### Prerequisites + +You need to have a breakwater server running using `cargo run --release -- --shared-memory-name breakwater --vnc`. +This opens the shared memory region for the breakwater-f-stack server to write into. +You can connect via VNC or append `--native-display` to the breakwater call to get a graphical output. + +#### Run breakwater-f-stack + +Start server on 0000:02:00.0: + +`sudo modprobe uio_pci_generic` + +`sudo dpdk-devbind.py --bind=uio_pci_generic 0000:02:00.0` + +`sudo bash -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'` + +As we linked against the dynamic library of `breakwater-parser-c-bindings`, we need to specify the `LD_LIBRARY_PATH` here: + +`sudo LD_LIBRARY_PATH=../target/release build/breakwater-f-stack` + +Add clients IP: + +`sudo ip link set up dev enp1s0f1` + +`sudo ip a a 10.0.0.42/8 dev enp1s0f1` + +`sudo ip a a 192.168.1.3/24 dev enp1s0f1` + +`ping 192.168.1.2` should now succeed (if it doesn't check e.g. `dmesg`). + +With a single desktop core from 2011 we can get 4.5 Gbit/s, pretty slow! +Normal breakwater on a single core on the same machine reaches 12G via loopback. + +### Run on multiple cores + +Disclaimer: While I got it to run on multiple cores I could not get it faster than running on a single core yet! + +1. Edit `lcore_mask` in `config.ini`. Hint: It's hexadecimal. +2. Start multiple processes using `sudo ./start.sh` +3. Stop all running processes using `sudo pkill -f breakwater-f-stack` + +### Special experiment for virtual device, as my NIC is not supported + +f-stack patches: + +```patch +diff --git a/lib/ff_config.c b/lib/ff_config.c +index 18380919..a9d74099 100644 +--- a/lib/ff_config.c ++++ b/lib/ff_config.c +@@ -1009,6 +1009,9 @@ dpdk_args_setup(struct ff_config *cfg) + + } + ++ // dpdk_argv[n++] = strdup("--vdev=net_pcap0,iface=lo"); ++ dpdk_argv[n++] = strdup("--vdev=net_tap"); ++ + if (cfg->dpdk.nb_vdev) { + for (i=0; idpdk.nb_vdev; i++) { + sprintf(temp, "--vdev=virtio_user%d,path=%s", +diff --git a/lib/ff_dpdk_if.c b/lib/ff_dpdk_if.c +index f6fe50e5..a3c17e74 100644 +--- a/lib/ff_dpdk_if.c ++++ b/lib/ff_dpdk_if.c +@@ -668,8 +668,8 @@ init_port_start(void) + printf("Use symmetric Receive-side Scaling(RSS) key\n"); + rsskey = symmetric_rsskey; + } +- port_conf.rx_adv_conf.rss_conf.rss_key = rsskey; +- port_conf.rx_adv_conf.rss_conf.rss_key_len = rsskey_len; ++ // port_conf.rx_adv_conf.rss_conf.rss_key = rsskey; ++ // port_conf.rx_adv_conf.rss_conf.rss_key_len = rsskey_len; + port_conf.rx_adv_conf.rss_conf.rss_hf &= dev_info.flow_type_rss_offloads; + if (port_conf.rx_adv_conf.rss_conf.rss_hf != + RTE_ETH_RSS_PROTO_MASK) { +``` + +`sudo ip a a 192.168.1.3/24 dev dtap0` diff --git a/breakwater-f-stack/breakwater-f-stack.c b/breakwater-f-stack/breakwater-f-stack.c new file mode 100644 index 0000000..f0c68ea --- /dev/null +++ b/breakwater-f-stack/breakwater-f-stack.c @@ -0,0 +1,314 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ff_config.h" +#include "ff_api.h" + +#include "breakwater-f-stack.h" +#include "framebuffer.h" +#include "parser.h" + +// For docs see the Rust docs in `breakwater-parser-c-bindings/src/lib.rs` +extern void breakwater_init_original_parser(int width, int height, char shared_memory_name[]); +extern size_t breakwater_original_parser_parser_lookahead(); +extern size_t breakwater_original_parser_parse( + const char* buffer, + size_t buffer_len, + unsigned char** out_response_ptr, + size_t* out_response_len +); + +#define MAX_EVENTS 512 + +/* kevent set */ +struct kevent kevSet; +/* events */ +struct kevent events[MAX_EVENTS]; +/* kq */ +int kq; +int sockfd; +#ifdef INET6 +int sockfd6; +#endif + +// The main read buffer +char buf[1024 * 1024]; + +size_t parser_lookahead; + +// Pointers for the response to the client +unsigned char* response = NULL; +size_t response_len = 0; + +// Array of pointers to client structures +client_state **clients; +// Max number of FDs (from ulimit -n) +size_t max_clients; + + +// Get the system limit for file descriptors +size_t get_max_fds() { + struct rlimit limit; + if (getrlimit(RLIMIT_NOFILE, &limit) != 0) { + perror("getrlimit failed"); + exit(EXIT_FAILURE); + } + return limit.rlim_cur; +} + +// Initialize client array +void init_clients() { + max_clients = get_max_fds(); + clients = calloc(max_clients, sizeof(client_state *)); + if (!clients) { + perror("Memory allocation for clients failed"); + exit(EXIT_FAILURE); + } + printf("Allocated space for %zu client connections (~%zu KB)\n", + max_clients, (max_clients * sizeof(client_state *)) / 1024); +} + +// Add a client state +void add_client(int fd) { + if (fd < 0 || fd >= max_clients) { + fprintf(stderr, "Invalid fd: %d\n", fd); + return; + } + + if (clients[fd] == NULL) { + clients[fd] = malloc(sizeof(client_state)); + if (!clients[fd]) { + perror("Failed to allocate client state"); + return; + } + memset(clients[fd], 0, sizeof(client_state)); + } +} + +// Lookup client state by fd +client_state *get_client(int fd) { + // if (fd < 0 || fd >= max_clients) { + // return NULL; + // } + return clients[fd]; +} + +// Remove a client state +void remove_client(int fd) { + if (fd < 0 || fd >= max_clients || clients[fd] == NULL) { + return; + } + free(clients[fd]); + clients[fd] = NULL; +} + +// Cleanup memory +void cleanup_clients() { + for (size_t i = 0; i < max_clients; i++) { + if (clients[i]) { + free(clients[i]); + } + } + free(clients); +} + + +int loop(void *arg) +{ + struct framebuffer *framebuffer = (struct framebuffer *)arg; + + /* Wait for events to happen */ + int nevents = ff_kevent(kq, NULL, 0, events, MAX_EVENTS, NULL); + int i; + + if (nevents < 0) { + printf("ff_kevent failed:%d, %s\n", errno, strerror(errno)); + return -1; + } + + for (i = 0; i < nevents; ++i) { + struct kevent event = events[i]; + int clientfd = (int)event.ident; + + /* Handle disconnect */ + if (event.flags & EV_EOF) { + /* Simply close socket */ + ff_close(clientfd); +#ifdef INET6 + } else if (clientfd == sockfd || clientfd == sockfd6) { +#else + } else if (clientfd == sockfd) { +#endif + int available = (int)event.data; + do { + int nclientfd = ff_accept(clientfd, NULL, NULL); + if (nclientfd < 0) { + printf("ff_accept failed:%d, %s\n", errno, strerror(errno)); + break; + } + + // printf("Got new client connection"); + + // Add to clients array + add_client(nclientfd); + // Add to event list + EV_SET(&kevSet, nclientfd, EVFILT_READ, EV_ADD, 0, 0, NULL); + + if(ff_kevent(kq, &kevSet, 1, NULL, 0, NULL) < 0) { + printf("ff_kevent error:%d, %s\n", errno, strerror(errno)); + return -1; + } + + available--; + } while (available); + } else if (event.filter == EVFILT_READ) { + ssize_t readlen = ff_read(clientfd, buf, sizeof(buf)); + // printf("readlen: %ld\n", readlen); + // client_state *client = get_client(clientfd); + // client->bytes_parsed += readlen; + + // I have hand-written same *very* basic and inperformant C parser. Because it was so + // slow, we are instead calling out to the breakwater-parser-c-bindings. + // size_t bytes_parsed = parse(buf, readlen, framebuffer, clientfd); + + // FIXME: Currently we don't keep any bytes left over from the previous loop iteration, + // which results in a few bytes getting missed. + // breakwater handles this correctly, for this implementation it's still needed as we + // have not commited to it yet. At least we already have the "client_state" for that + // already, so please feel free to implement it. + + // We need to make sure the parser has enough lookahead space + if (readlen + parser_lookahead > sizeof(buf)) { + // Read as many bytes as we can safely do + readlen = sizeof(buf) - parser_lookahead; + } else { + // Fill the next parser_lookahead bytes with zero, so we don't accidentally + // fabricate a valid, but semantically wrong command (e.g. resulting in + // "ghost pixels") + memset(&buf[readlen], 0, parser_lookahead); + // Increase the readlen, so that parser parses everything + readlen += parser_lookahead; + } + // printf("readlen after truncation: %ld\n", readlen); + + long parsed = breakwater_original_parser_parse(buf, readlen, &response, &response_len); + // printf("Parsed %ld bytes\n", parsed); + + // Write the response to the client + ff_write(clientfd, response, response_len); + } else { + printf("unknown event: %8.8X\n", event.flags); + } + } + + return 0; +} + +int main(int argc, char * argv[]) +{ + int err = 0; + + breakwater_init_original_parser(WIDTH, HEIGHT, SHARED_MEMORY_NAME); + parser_lookahead = breakwater_original_parser_parser_lookahead(); + + struct framebuffer* framebuffer; + if((err = create_fb(&framebuffer, WIDTH, HEIGHT, SHARED_MEMORY_NAME))) { + fprintf(stderr, "Failed to allocate framebuffer: %s\n", strerror(err)); + return err; + } + + // for (uint16_t x = 0; x <= 150; x++) { + // for (uint16_t y = 0; y <= 50; y++) { + // fb_set(framebuffer, x, y, 0x00ff0000); + // } + // } + + ff_init(argc, argv); + + kq = ff_kqueue(); + if (kq < 0) { + printf("ff_kqueue failed, errno:%d, %s\n", errno, strerror(errno)); + exit(1); + } + + sockfd = ff_socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + printf("ff_socket failed, sockfd:%d, errno:%d, %s\n", sockfd, errno, strerror(errno)); + exit(1); + } + + /* Set non blocking */ + int on = 1; + ff_ioctl(sockfd, FIONBIO, &on); + + struct sockaddr_in my_addr; + bzero(&my_addr, sizeof(my_addr)); + my_addr.sin_family = AF_INET; + my_addr.sin_port = htons(SERVER_PORT); + my_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + int ret = ff_bind(sockfd, (struct linux_sockaddr *)&my_addr, sizeof(my_addr)); + if (ret < 0) { + printf("ff_bind failed, sockfd:%d, errno:%d, %s\n", sockfd, errno, strerror(errno)); + exit(1); + } + + ret = ff_listen(sockfd, MAX_EVENTS); + if (ret < 0) { + printf("ff_listen failed, sockfd:%d, errno:%d, %s\n", sockfd, errno, strerror(errno)); + exit(1); + } + + EV_SET(&kevSet, sockfd, EVFILT_READ, EV_ADD, 0, MAX_EVENTS, NULL); + /* Update kqueue */ + ff_kevent(kq, &kevSet, 1, NULL, 0, NULL); + +#ifdef INET6 + sockfd6 = ff_socket(AF_INET6, SOCK_STREAM, 0); + if (sockfd6 < 0) { + printf("ff_socket failed, sockfd6:%d, errno:%d, %s\n", sockfd6, errno, strerror(errno)); + exit(1); + } + + struct sockaddr_in6 my_addr6; + bzero(&my_addr6, sizeof(my_addr6)); + my_addr6.sin6_family = AF_INET6; + my_addr6.sin6_port = htons(SERVER_PORT); + my_addr6.sin6_addr = in6addr_any; + + ret = ff_bind(sockfd6, (struct linux_sockaddr *)&my_addr6, sizeof(my_addr6)); + if (ret < 0) { + printf("ff_bind failed, sockfd6:%d, errno:%d, %s\n", sockfd6, errno, strerror(errno)); + exit(1); + } + + ret = ff_listen(sockfd6, MAX_EVENTS); + if (ret < 0) { + printf("ff_listen failed, sockfd6:%d, errno:%d, %s\n", sockfd6, errno, strerror(errno)); + exit(1); + } + + EV_SET(&kevSet, sockfd6, EVFILT_READ, EV_ADD, 0, MAX_EVENTS, NULL); + ret = ff_kevent(kq, &kevSet, 1, NULL, 0, NULL); + if (ret < 0) { + printf("ff_kevent failed:%d, %s\n", errno, strerror(errno)); + exit(1); + } +#endif + + init_clients(); + ff_run(loop, framebuffer); + + cleanup_clients(); + return 0; +} diff --git a/breakwater-f-stack/breakwater-f-stack.h b/breakwater-f-stack/breakwater-f-stack.h new file mode 100644 index 0000000..9429b56 --- /dev/null +++ b/breakwater-f-stack/breakwater-f-stack.h @@ -0,0 +1,5 @@ +// TODO: Read via CLI args +#define SERVER_PORT 1234 +#define WIDTH 1280 +#define HEIGHT 720 +#define SHARED_MEMORY_NAME "breakwater" diff --git a/breakwater-f-stack/config.ini b/breakwater-f-stack/config.ini new file mode 100644 index 0000000..b9fd01e --- /dev/null +++ b/breakwater-f-stack/config.ini @@ -0,0 +1,300 @@ +# Copied and modified from https://github.com/F-Stack/f-stack/blob/dev/config.ini + +[dpdk] +# Hexadecimal bitmask of cores to run on (without trailing 0x) +lcore_mask=1 + +# Number of memory channels. +channel=4 + +# Specify base virtual address to map. +#base_virtaddr=0x7f0000000000 + +# Promiscuous mode of nic, defualt: enabled. +promiscuous=1 +numa_on=1 + +# TX checksum offload skip, default: disabled. +# We need this switch enabled in the following cases: +# -> The application want to enforce wrong checksum for testing purposes +# -> Some cards advertize the offload capability. However, doesn't calculate checksum. +tx_csum_offoad_skip=0 + +# TCP segment offload, default: disabled. +tso=0 + +# HW vlan strip, default: enabled. +vlan_strip=1 + +# Set [vlanN]'s addrs like [portN] later +# the format is same as port_list +# Set vlan filter id, to enable L3/L4 RSS below vlan hdr is not enable after f-stack-1.22. +vlan_filter=1,2,4-6 + +# sleep when no pkts incomming +# unit: microseconds +idle_sleep=0 + +# sent packet delay time(0-100) while send less than 32 pkts. +# default 100 us. +# if set 0, means send pkts immediately. +# if set >100, will dealy 100 us. +# unit: microseconds +pkt_tx_delay=100 + +# use symmetric Receive-side Scaling(RSS) key, default: disabled. +symmetric_rss=0 + +# PCI device enable list. +# And driver options +#allow=02:00.0 +# for multiple PCI devices +#allow=02:00.0,03:00.0 + +# enabled port list +# +# EBNF grammar: +# +# exp ::= num_list {"," num_list} +# num_list ::= | +# range ::= "-" +# num ::= '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' +# +# examples +# 0-3 ports 0, 1,2,3 are enabled +# 1-3,4,7 ports 1,2,3,4,7 are enabled +# +# If use bonding, shoule config the bonding port id in port_list +# and not config slave port id in port_list +# such as, port 0 and port 1 trank to a bonding port 2, +# should set `port_list=2` and config `[port2]` section + +port_list=0 + +# Number of vdev. +nb_vdev=0 + +# Number of bond. +nb_bond=0 + +# log level for dpdk, optional +# log_level=0 + +# Each core write into own pcap file, which is open one time, close one time if enough. +# Support dump the first snaplen bytes of each packet. +# if pcap file is lager than savelen bytes, it will be closed and next file was dumped into. +[pcap] +enable=0 +snaplen=96 +savelen=16777216 +savepath=. + +# Port config section +# Correspond to dpdk.port_list's index: port0, port1... +[port0] +#addr=10.0.0.2 +#netmask=255.0.0.0 +#broadcast=10.255.255.255 +#gateway=10.0.0.1 +addr=192.168.1.2 +netmask=255.255.255.0 +broadcast=192.168.1.255 +gateway=192.168.1.1 +# set interface name, Optional parameter. +#if_name=eno7 + +# IPv6 net addr, Optional parameters. +#addr6=ff::02 +#prefix_len=64 +#gateway6=ff::01 + +# Multi virtual IPv4/IPv6 net addr, Optional parameters. +# `vip_ifname`: default `f-stack-x` +# `vip_addr`: Separated by semicolons, MAX number 64; +# Only support netmask 255.255.255.255, broadcast x.x.x.255 now, hard code in `ff_veth_setvaddr`. +# `ipfw_pr`: Set simplest policy routing, Optional parameters. +# Such as the cmd `ff_ipfw -P 0 add 100 setfib 0 ip from 192.168.0.0/24 to any out` +# can set parameter`192.168.0.0 255.255.255.0`, cidr and netmask separated by space. +# Multi cidr separated by semicolons. +# IPv4 only now, and if you want set more complex policy routing, should use tool `ff_ipfw`. +# `vip_addr6`: Separated by semicolons, MAX number 64. +# `vip_prefix_len`: All addr6 use the same prefix now, default 64. +#vip_ifname=lo0 +#vip_addr=192.168.0.3;192.168.0.4;192.168.0.5;192.168.0.6 +#ipfw_pr=192.168.0.0 255.255.255.0;192.168.10.0 255.255.255.0 +#vip_addr6=ff::03;ff::04;ff::05;ff::06;ff::07 +#vip_prefix_len=64 + +# lcore list used to handle this port +# the format is same as port_list +#lcore_list=0 + +# bonding slave port list used to handle this port +# need to config while this port is a bonding port +# the format is same as port_list +#slave_port_list=0,1 + +# Vlan config section, Must set after all [portN] +# NOTE1: Must enable dpdk.vlan_filter first, and match it. +# NOTE2: If enable vlan config, all [PortN] config will be ignored! +#[vlan1] +#portid=0 +#addr=192.169.0.2 +#netmask=255.255.255.0 +#broadcast=192.169.0.255 +#gateway=192.169.0.1 +# +#vip_addr=192.169.0.3;192.169.0.4;192.169.0.5;192.169.0.6 +#ipfw_pr=192.169.0.0 255.255.255.0;192.169.10.0 255.255.255.0 +# +#[vlan2] +#portid=0 +#addr=192.169.1.2 +#netmask=255.255.255.0 +#broadcast=192.169.1.255 +#gateway=192.169.1.1 +# +#vip_addr=192.169.1.3;192.169.1.4;192.169.1.5;192.169.1.6 +#ipfw_pr=192.169.1.0 255.255.255.0;192.169.11.0 255.255.255.0 +# +#[vlan4] +#portid=0 +#addr=192.169.2.2 +#netmask=255.255.255.0 +#broadcast=192.169.2.255 +#gateway=192.169.2.1 +# +#vip_addr=192.169.2.3;192.169.2.4;192.169.2.5;192.169.2.6 +#ipfw_pr=192.169.2.0 255.255.255.0;192.169.12.0 255.255.255.0 +# +#[vlan5] +#portid=0 +#addr=192.169.3.2 +#netmask=255.255.255.0 +#broadcast=192.169.3.255 +#gateway=192.169.3.1 +# +#addr6=fe::32 +#prefix_len=64 +#gateway6=fe::31 +# +#vip_addr=192.169.3.3;192.169.3.4;192.169.3.5;192.169.3.6 +#ipfw_pr=192.169.3.0 255.255.255.0;192.169.13.0 255.255.255.0 +#vip_addr6=fe::33;fe::34;fe::35;fe::36;fe::37 +#vip_prefix_len=64 + +# Vdev config section +# orrespond to dpdk.nb_vdev's index: vdev0, vdev1... +# iface : Shouldn't set always. +# path : The vuser device path in container. Required. +# queues : The max queues of vuser. Optional, default 1, greater or equal to the number of processes. +# queue_size : Queue size.Optional, default 256. +# mac : The mac address of vuser. Optional, default random, if vhost use phy NIC, it should be set to the phy NIC's mac. +# cq : Optional, if queues = 1, default 0; if queues > 1 default 1. +#[vdev0] +##iface=/usr/local/var/run/openvswitch/vhost-user0 +#path=/var/run/openvswitch/vhost-user0 +#queues=1 +#queue_size=256 +#mac=00:00:00:00:00:01 +#cq=0 + +# bond config section +# See http://doc.dpdk.org/guides/prog_guide/link_bonding_poll_mode_drv_lib.html +#[bond0] +#mode=4 +#slave=0000:0a:00.0,slave=0000:0a:00.1 +#primary=0000:0a:00.0 +#mac=f0:98:38:xx:xx:xx +## opt argument +#socket_id=0 +#xmit_policy=l23 +#lsc_poll_period_ms=100 +#up_delay=10 +#down_delay=50 + +# Kni config: if enabled and method=reject, +# all packets that do not belong to the following tcp_port and udp_port +# will transmit to kernel; if method=accept, all packets that belong to +# the following tcp_port and udp_port will transmit to kernel. +# type: exception path type, 0 means kni(must set meson -Ddisable_libs=flow_classif to re-enable kni in DPDK first), 1 means virtio_user(linux only) +#[kni] +#type=1 +#enable=1 +#method=reject +# The format is same as port_list +#tcp_port=80,443 +#udp_port=53 +# KNI ratelimit value, default: 0, means disable ratelimit. +# example: +# The total speed limit for a single process entering the kni ring is 10,000 QPS, +# 1000 QPS for general packets, 9000 QPS for console packets (ospf/arp, etc.) +# The total speed limit for kni forwarding to the kernel is 20,000 QPS. +#console_packets_ratelimit=0 +#general_packets_ratelimit=0 +#kernel_packets_ratelimit=0 + +# FreeBSD network performance tuning configurations. +# Most native FreeBSD configurations are supported. +[freebsd.boot] +# If use rack/bbr which depend HPTS, you should set a greater value of hz, such as 1000000 means a tick is 1us. +hz=100 + +# Block out a range of descriptors to avoid overlap +# with the kernel's descriptor space. +# You can increase this value according to your app. +fd_reserve=1024 + +kern.ipc.maxsockets=262144 + +net.inet.tcp.syncache.hashsize=4096 +net.inet.tcp.syncache.bucketlimit=100 + +net.inet.tcp.tcbhashsize=65536 + +kern.ncallout=262144 + +kern.features.inet6=1 + +[freebsd.sysctl] +kern.ipc.somaxconn=32768 +kern.ipc.maxsockbuf=16777216 + +net.add_addr_allfibs=1 + +net.link.ether.inet.maxhold=5 + +net.inet.tcp.fast_finwait2_recycle=1 +net.inet.tcp.sendspace=16384 +net.inet.tcp.recvspace=8192 +#net.inet.tcp.nolocaltimewait=1 +net.inet.tcp.cc.algorithm=cubic +net.inet.tcp.sendbuf_max=16777216 +net.inet.tcp.recvbuf_max=16777216 +net.inet.tcp.sendbuf_auto=1 +net.inet.tcp.recvbuf_auto=1 +net.inet.tcp.sendbuf_inc=16384 +#net.inet.tcp.recvbuf_inc=524288 +net.inet.tcp.sack.enable=1 +net.inet.tcp.blackhole=1 +net.inet.tcp.msl=2000 +net.inet.tcp.delayed_ack=1 +net.inet.tcp.rfc1323=1 + +net.inet.udp.blackhole=1 +net.inet.ip.redirect=0 +net.inet.ip.forwarding=0 + +net.inet6.ip6.auto_linklocal=1 +net.inet6.ip6.accept_rtadv=2 +net.inet6.icmp6.rediraccept=1 +net.inet6.ip6.forwarding=0 + +# set default stacks:freebsd, rack or bbr, may be you need increase the value of parameter 'freebsd.boot.hz' while use rack or bbr. +net.inet.tcp.functions_default=freebsd +# need by bbr, should enable it. +net.inet.tcp.hpts.skip_swi=1 +# Interval between calls to hpts_timeout_dir. default min 250us, max 256-512ms, default 512ms. +net.inet.tcp.hpts.minsleep=250 +# [25600-51200] +net.inet.tcp.hpts.maxsleep=51200 diff --git a/breakwater-f-stack/default.nix b/breakwater-f-stack/default.nix new file mode 100644 index 0000000..597e51a --- /dev/null +++ b/breakwater-f-stack/default.nix @@ -0,0 +1,22 @@ +{ + nixpkgs ? import {}, + nixpkgsUnstable ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/nixos-unstable.tar.gz") {}, +}: + +nixpkgs.mkShell { + buildInputs = [ + nixpkgs.pkg-config + nixpkgs.dpdk + # nixpkgsUnstable.dpdk # Uncomment to use dpdk from nixpkgs-unstable + nixpkgs.openssl + + nixpkgs.numactl + nixpkgs.zlib + nixpkgs.jemalloc + nixpkgs.jansson + nixpkgs.libpcap + nixpkgs.libnl + nixpkgs.libelf + # nixpkgs.libnfnetlink + ]; +} diff --git a/breakwater-f-stack/framebuffer.c b/breakwater-f-stack/framebuffer.c new file mode 100644 index 0000000..91bf695 --- /dev/null +++ b/breakwater-f-stack/framebuffer.c @@ -0,0 +1,84 @@ +// Partly copied from https://github.com/sbernauer/pixelflut-v6/blob/bdc32037c6910846f3b378dccf97acdb304d683f/dpdk-server/framebuffer.c, +// go look there for more features, such as statistics! + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "framebuffer.h" + +int create_fb(struct framebuffer** framebuffer, uint16_t width, uint16_t height, char* shared_memory_name) { + int fd = shm_open(shared_memory_name, O_RDWR, 0666); + if(fd == -1) { + printf("Failed to open shared memory with name %s: %s\n", shared_memory_name, strerror(errno)); + return errno; + } + + struct stat shared_memory_stats; + if (fstat(fd, &shared_memory_stats) == -1) { + printf("Failed to fstat the shared memory with name %s: %s\n", shared_memory_name, strerror(errno)); + return errno; + } + + int expected_shared_memory_size = 2 * sizeof(uint16_t) /* size header */ + + width * height * sizeof(uint32_t) /* pixels */; + + if (shared_memory_stats.st_size != expected_shared_memory_size) { + printf("Found existing shared memory with size of %lu bytes. However, I expected it to be of size %u, as the" + "framebuffer has (%u, %u) pixels. The Pixelflut backend and frontend seem to use different resolutions! " + "In case you want to re-size your existing framebuffer please execute 'rm /dev/shm%s'\n", + shared_memory_stats.st_size, expected_shared_memory_size, width, height, shared_memory_name); + return EINVAL; + } else { + printf("Using existing shared memory of correct size\n"); + } + + char* shared_memory; + shared_memory = mmap(NULL, expected_shared_memory_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (shared_memory == MAP_FAILED) { + printf("Failed to mmap the the shared memory with name %s: %s\n", shared_memory_name, strerror(errno)); + return errno; + } + + // We need to set width and height so that other tools (e.g. the frontend) can detect the framebuffer size + uint16_t* width_ptr = (uint16_t*)shared_memory; + if (*width_ptr != width) { + printf("Found existing shared memory, but it has the width %u, while I expected %u\n", *width_ptr, width); + return EINVAL; + } + + uint16_t* height_ptr = (uint16_t*)(shared_memory + 2); + if (*height_ptr != height) { + printf("Found existing shared memory, but it has the height %u, while I expected %u\n", *height_ptr, height); + return EINVAL; + } + + struct framebuffer* fb = malloc(sizeof(struct framebuffer)); + fb->width = width; + fb->height = height; + fb->pixels = (uint32_t*)(shared_memory + 2 * sizeof(uint16_t) /* size header */); + + printf("Created framebuffer of size (%u,%u) backed by shared memory with the name %s\n", + width, height, shared_memory_name); + + *framebuffer = fb; + return 0; +} + +// Only sets pixel if it is within bounds +void fb_set(struct framebuffer* framebuffer, uint16_t x, uint16_t y, uint32_t rgba) { + if (x < framebuffer->width && y < framebuffer->height) { + framebuffer->pixels[x + y * framebuffer->width] = rgba; + } +} + +// Does *not* check for bounds +uint32_t fb_get(struct framebuffer* framebuffer, uint16_t x, uint16_t y) { + return framebuffer->pixels[x + y * framebuffer->width]; +} diff --git a/breakwater-f-stack/framebuffer.h b/breakwater-f-stack/framebuffer.h new file mode 100644 index 0000000..6cee031 --- /dev/null +++ b/breakwater-f-stack/framebuffer.h @@ -0,0 +1,15 @@ +#ifndef _FRAMEBUFFER_H_ +#define _FRAMEBUFFER_H_ + +struct framebuffer { + uint16_t width; + uint16_t height; + + uint32_t* pixels; +}; + +int create_fb(struct framebuffer** framebuffer, uint16_t width, uint16_t height, char* shared_memory_name); +void fb_set(struct framebuffer* framebuffer, uint16_t x, uint16_t y, uint32_t rgba); +uint32_t fb_get(struct framebuffer* framebuffer, uint16_t x, uint16_t y); + +#endif diff --git a/breakwater-f-stack/parser.c b/breakwater-f-stack/parser.c new file mode 100644 index 0000000..13c0f1a --- /dev/null +++ b/breakwater-f-stack/parser.c @@ -0,0 +1,117 @@ +// Port of breakwater-parser/src/original.rs to C +// Yes, I know this is ugly +// Yes, I only care about performance :) +// PRs welcome to improve the situation without compromising on the performance! + +#include +#include +#include + +#include "ff_api.h" + +#include "breakwater-f-stack.h" +#include "framebuffer.h" +#include "parser.h" + +#define HELP_TEXT "Pixelflut server, see https://github.com/sbernauer/breakwater/ and https://wiki.cccgoe.de/wiki/Pixelflut\n" + +// Fast ASCII to int (no error checking) +static inline uint16_t fast_atoi(const char **p) { + int v = 0; + while (**p >= '0' && **p <= '9') { + v = v * 10 + (**p - '0'); + (*p)++; + } + return v; +} + +// Fast hex to int (handles rrggbb or rrggbbaa) +static inline uint32_t fast_hex(const char *p, int len) { + uint32_t v = 0; + for (int i = 0; i < len; i++) { + v <<= 4; + char c = *p++; + if (c >= '0' && c <= '9') v |= (c - '0'); + else if (c >= 'a' && c <= 'f') v |= (c - 'a' + 10); + else if (c >= 'A' && c <= 'F') v |= (c - 'A' + 10); + else break; + } + return v; +} + +size_t parse(const char *buffer, size_t length, struct framebuffer* framebuffer, int clientfd) { + const char *p = buffer; + const char *end = p + length; + + while (p < end) { + if (memcmp(p, "PX ", 3) == 0) { + p += 3; + int x = fast_atoi(&p); + if (*p != ' ') { + p += 1; + continue; + } + p += 1; + int y = fast_atoi(&p); + + // Request out of screen bounds + if (x >= WIDTH || y >= HEIGHT) { + continue; + } + + // Command to set pixel + if (*p == ' ') { + p += 1; + uint32_t rgb = fast_hex(p, 6); + p += 6; + + rgb = + // Green + rgb & 0x0000ff00 + // Red + | ((rgb >> 16) & 0x000000ff) + // Blue + | ((rgb << 16) & 0x00ff0000); + + fb_set(framebuffer, x, y, rgb); + continue; + } + + // Command to read pixel + else if (*p == '\n') { + p += 1; + uint32_t rgb = fb_get(framebuffer, x, y); + + char out[32]; + int len = snprintf(out, sizeof(out), "PX %d %d %06x\n", x, y, rgb); + ff_write(clientfd, out, len); + continue; + } + + else { + // The parsing already moved p + continue; + } + } + + else if (memcmp(p, "SIZE", 4) == 0) { + char out[32]; + int len = snprintf(out, sizeof(out), "SIZE %d %d\n", WIDTH, HEIGHT); + ff_write(clientfd, out, len); + + p += 4; + continue; + } + + else if (memcmp(p, "HELP", 4) == 0) { + ff_write(clientfd, HELP_TEXT, sizeof(HELP_TEXT) - 1); + + p += 4; + continue; + } + + p++; + } + + return length; +} diff --git a/breakwater-f-stack/parser.h b/breakwater-f-stack/parser.h new file mode 100644 index 0000000..a9a9111 --- /dev/null +++ b/breakwater-f-stack/parser.h @@ -0,0 +1,20 @@ +#ifndef _PARSER_H_ +#define _PARSER_H_ + +#include + +#include "framebuffer.h" + +// Longest possible command +#define PARSER_LOOKAHEAD (sizeof("PX 1234 1234 rrggbbaa\n") - 1) // Excludes null terminator + +typedef struct { + size_t leftover_bytes; + char leftover[PARSER_LOOKAHEAD]; + long long bytes_parsed; +} client_state; + +// Returns the last byte parsed. The next parsing loop will again contain all data that was not parsed. +size_t parse(const char *buffer, size_t length, struct framebuffer* framebuffer, int socket); + +#endif diff --git a/breakwater-f-stack/start.sh b/breakwater-f-stack/start.sh new file mode 100755 index 0000000..eb55f72 --- /dev/null +++ b/breakwater-f-stack/start.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copied from https://github.com/F-Stack/f-stack/blob/dev/start.sh + +function usage() { + echo "F-Stack app start tool" + echo "Options:" + echo " -c [conf] Path of config file" + echo " -b [N] Path of binary" + echo " -o [N] Other ARGs for app" + echo " -h show this help" + exit +} + +conf=config.ini +bin=./build/breakwater-f-stack + +while getopts "c:b:o:h" args +do + case $args in + c) + conf=$OPTARG + ;; + b) + bin=$OPTARG + ;; + o) + others=$OPTARG + ;; + h) + usage + exit 0 + ;; + esac +done + +if ! type "bc" > /dev/null 2>&1; then + echo "please install bc" + exit +fi + +allcmask0x=`cat ${conf}|grep lcore_mask|awk -F '=' '{print $2}'` +((allcmask=16#$allcmask0x)) + +num_procs=0 +PROCESSOR=$(grep 'processor' /proc/cpuinfo |sort |uniq |wc -l) +for((i=0;i<${PROCESSOR};++i)) +do + mask=`echo "2^$i"|bc` + ((result=${allcmask} & ${mask})) + if [ ${result} != 0 ] + then + ((num_procs++)); + fi +done + +for((proc_id=0; proc_id<${num_procs}; ++proc_id)) +do + if ((proc_id == 0)) + then + echo "${bin} --conf ${conf} --proc-type=primary --proc-id=${proc_id} ${others}" + ${bin} --conf ${conf} --proc-type=primary --proc-id=${proc_id} ${others} & + sleep 5 + else + echo "${bin} --conf ${conf} --proc-type=secondary --proc-id=${proc_id} ${others}" + ${bin} --conf ${conf} --proc-type=secondary --proc-id=${proc_id} ${others} & + fi +done + +echo +echo +echo "Use the following command to stop the processes again (assuming you are using the default binary)" +echo "sudo pkill -f breakwater-f-stack"