diff --git a/src/utils/qemu.c b/src/utils/qemu.c index 29a0b16cf..8462c22f4 100644 --- a/src/utils/qemu.c +++ b/src/utils/qemu.c @@ -67,15 +67,15 @@ int pg_util_ssh(const char *host, } int pg_util_spawn_qemu(const char *socket_path_0, - const char *socket_path_1, - const char *mac_0, - const char *mac_1, - const int is_server_0, - const int is_server_1, - const char *vm_image_path, - const char *vm_key_path, - const char *hugepages_path, - struct pg_error **errp) + const char *socket_path_1, + const char *mac_0, + const char *mac_1, + const int is_server_0, + const int is_server_1, + const char *vm_image_path, + const char *vm_key_path, + const char *hugepages_path, + struct pg_error **errp) { int child_pid = 0; static uint16_t vm_id; @@ -83,11 +83,12 @@ int pg_util_spawn_qemu(const char *socket_path_0, pg_autofree char *argv_qemu = NULL; const char *argv_sock_0 = ""; const char *argv_sock_1 = ""; - const char* server_params = ",server,nowait"; + const char *server_params = ",server,nowait"; pg_autofree char *argv_sock_0_t = NULL; pg_autofree char *argv_sock_1_t = NULL; pg_autofree char *ssh_cmd = NULL; GError *error = NULL; + if(!is_server_1) g_assert(g_file_test(socket_path_0, G_FILE_TEST_EXISTS)); if (socket_path_1 && !is_server_1) @@ -102,7 +103,9 @@ int pg_util_spawn_qemu(const char *socket_path_0, " -netdev type=vhost-user,id=mynet0,", "chardev=char0,vhostforce", " -device virtio-net-pci,mac=%s", - ",netdev=mynet0"), socket_path_0, is_server_0 ? server_params : "", mac_0); + ",netdev=mynet0"), socket_path_0, + is_server_0 ? server_params : "", + mac_0); argv_sock_0 = argv_sock_0_t; } @@ -112,7 +115,9 @@ int pg_util_spawn_qemu(const char *socket_path_0, " -netdev type=vhost-user,id=mynet1,", "chardev=char1,vhostforce", " -device virtio-net-pci,mac=%s", - ",netdev=mynet1"), socket_path_1, is_server_0 ? server_params : "", mac_1); + ",netdev=mynet1"), socket_path_1, + is_server_0 ? server_params : "", + mac_1); argv_sock_1 = argv_sock_1_t; } @@ -136,10 +141,10 @@ int pg_util_spawn_qemu(const char *socket_path_0, argv = g_strsplit(argv_qemu, " ", 0); g_assert(g_spawn_async(NULL, argv, NULL, - (GSpawnFlags) G_SPAWN_SEARCH_PATH | - G_SPAWN_DO_NOT_REAP_CHILD, - (GSpawnChildSetupFunc) NULL, - NULL, &child_pid, &error)); + (GSpawnFlags) G_SPAWN_SEARCH_PATH | + G_SPAWN_DO_NOT_REAP_CHILD, + (GSpawnChildSetupFunc) NULL, + NULL, &child_pid, &error)); g_assert(!error); ssh_cmd = g_strdup_printf( diff --git a/src/utils/qemu.h b/src/utils/qemu.h index 54fc68ab9..a9bc7264b 100644 --- a/src/utils/qemu.h +++ b/src/utils/qemu.h @@ -29,15 +29,15 @@ int pg_util_ssh(const char *host, __attribute__((__format__(__printf__, 4, 5))); int pg_util_spawn_qemu(const char *socket_path_0, - const char *socket_path_1, - const char *mac_0, - const char *mac_1, + const char *socket_path_1, + const char *mac_0, + const char *mac_1, const int is_server_0, const int is_server_1, - const char *vm_image_path, - const char *vm_ssh_key_path, - const char *hugepages_path, - struct pg_error **errp); + const char *vm_image_path, + const char *vm_ssh_key_path, + const char *hugepages_path, + struct pg_error **errp); void pg_util_stop_qemu(int qemu_pid, int signal); diff --git a/src/vhost.c b/src/vhost.c index 7a84d5b37..a7f8922d8 100644 --- a/src/vhost.c +++ b/src/vhost.c @@ -117,7 +117,7 @@ static struct pg_brick_config *vhost_config_new(const char *name, { struct pg_brick_config *config = g_new0(struct pg_brick_config, 1); struct pg_vhost_config *vhost_config = g_new0(struct pg_vhost_config, - 1); + 1); config->brick_config = (void *) vhost_config; vhost_config->flags = flags; @@ -127,17 +127,18 @@ static struct pg_brick_config *vhost_config_new(const char *name, /* To know if a vhost brick is server or not. */ static int pg_vhost_is_server(struct pg_brick *brick) { - struct pg_vhost_state *state = - pg_brick_get_state(brick, struct pg_vhost_state); + struct pg_vhost_state *state = + pg_brick_get_state(brick, + struct pg_vhost_state); - if(state->flags & 1) - return 0; - return 1; + if(state->flags & 1) + return 0; + return -1; } static int vhost_burst(struct pg_brick *brick, enum pg_side from, - uint16_t edge_index, struct rte_mbuf **pkts, - uint64_t pkts_mask, struct pg_error **errp) + uint16_t edge_index, struct rte_mbuf **pkts, + uint64_t pkts_mask, struct pg_error **errp) { struct pg_vhost_state *state = pg_brick_get_state(brick, struct pg_vhost_state); @@ -149,24 +150,24 @@ static int vhost_burst(struct pg_brick *brick, enum pg_side from, /* Try lock */ if (unlikely(check_atomic && - !rte_atomic32_test_and_set(&state->allow_queuing))) + !rte_atomic32_test_and_set(&state->allow_queuing))) return 0; virtio_net = state->vid; pkts_count = pg_packets_pack(state->out, pkts, pkts_mask); if (pkts_count > 32) { bursted_pkts = rte_vhost_enqueue_burst(virtio_net, - VIRTIO_RXQ, - state->out, - 32); + VIRTIO_RXQ, + state->out, + 32); bursted_pkts += rte_vhost_enqueue_burst(virtio_net, VIRTIO_RXQ, state->out + 32, pkts_count - 32); } else { bursted_pkts = rte_vhost_enqueue_burst(virtio_net, - VIRTIO_RXQ, - state->out, - pkts_count); + VIRTIO_RXQ, + state->out, + pkts_count); } if (check_atomic) @@ -182,7 +183,7 @@ static int vhost_burst(struct pg_brick *brick, enum pg_side from, if (side->burst_count_cb != NULL) side->burst_count_cb(side->burst_count_private_data, - bursted_pkts); + bursted_pkts); #endif /* #ifdef PG_VHOST_BENCH */ return 0; } @@ -202,7 +203,7 @@ void pg_vhost_request_remove(struct pg_brick *brick) #endif /* PG_VHOST_FASTER_YET_BROKEN_POLL */ static int vhost_poll(struct pg_brick *brick, uint16_t *pkts_cnt, - struct pg_error **errp) + struct pg_error **errp) { struct pg_vhost_state *state = pg_brick_get_state(brick, struct pg_vhost_state); @@ -262,7 +263,7 @@ static int vhost_poll(struct pg_brick *brick, uint16_t *pkts_cnt, pkts_mask = pg_mask_firsts(count); ret = pg_brick_burst(s->edge.link, state->output, s->edge.pair_index, - in, pkts_mask, errp); + in, pkts_mask, errp); pg_packets_free(in, pkts_mask); return ret; } @@ -289,7 +290,7 @@ static enum pg_side vhost_get_side(struct pg_brick *brick) } struct pg_brick *pg_vhost_new(const char *name, uint64_t flags, - struct pg_error **errp) + struct pg_error **errp) { struct pg_brick_config *config = vhost_config_new(name, flags); struct pg_brick *ret = pg_brick_new("vhost", config, errp); @@ -367,13 +368,11 @@ static void on_destroy_device(int dev) rte_vhost_get_ifname(dev, buf, 256); printf("VM disconnecting from socket: %s\n", buf); - pthread_mutex_lock(&mutex); - LIST_FOREACH(s, &sockets, socket_list) { if (!strcmp(s->path, buf)) { while (!rte_atomic32_test_and_set( - &s->state->allow_queuing)) + &s->state->allow_queuing)) sched_yield(); #ifdef PG_VHOST_FASTER_YET_BROKEN_POLL s->state->check_atomic = 1024; @@ -383,9 +382,7 @@ static void on_destroy_device(int dev) break; } } - pthread_mutex_unlock(&mutex); - } static void vhost_create_socket(struct pg_vhost_state *state, uint64_t flags, struct pg_error **errp) @@ -397,17 +394,11 @@ static void vhost_create_socket(struct pg_vhost_state *state, uint64_t flags, path = g_strdup_printf("%s/qemu-%s", sockets_path, state->brick.name); /* If the socket is CLIENT do NOT destroy the socket. */ if((flags & 1) == 0) - { g_remove(path); - } - /*else - { - flags = flags | 31; - } - */ + printf("New vhost-user socket: %s, zero-copy %s\n", path, - (flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY) ? - "enable" : "disable"); + (flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY) ? + "enable" : "disable"); flags = flags & (RTE_VHOST_USER_CLIENT | RTE_VHOST_USER_NO_RECONNECT | RTE_VHOST_USER_DEQUEUE_ZERO_COPY); @@ -454,7 +445,7 @@ static void vhost_create_socket(struct pg_vhost_state *state, uint64_t flags, "vhost not ready, did you called vhost_start after packetgraph_start ?" static int vhost_init(struct pg_brick *brick, struct pg_brick_config *config, - struct pg_error **errp) + struct pg_error **errp) { struct pg_vhost_state *state; struct pg_vhost_config *vhost_config; @@ -501,7 +492,7 @@ static int vhost_init(struct pg_brick *brick, struct pg_brick_config *config, * */ static void check_and_store_base_dir(const char *base_dir, - struct pg_error **errp) + struct pg_error **errp) { char *resolved_path = g_malloc0(PATH_MAX); struct stat st; @@ -580,8 +571,8 @@ static struct pg_brick_ops vhost_ops = { .init = vhost_init, .destroy = vhost_destroy, - .link_notify = vhost_link, - .get_side = vhost_get_side, + .link_notify = vhost_link, + .get_side = vhost_get_side, .unlink = pg_brick_generic_unlink, .rx_bytes = rx_bytes, .tx_bytes = tx_bytes, diff --git a/tests/integration/tests.c b/tests/integration/tests.c index b026ce619..713353fc2 100644 --- a/tests/integration/tests.c +++ b/tests/integration/tests.c @@ -1,3 +1,4 @@ + /* Copyright 2015 Outscale SAS * * This file is part of Packetgraph. @@ -171,6 +172,7 @@ static inline int start_qemu_graph(struct branch *branch, int qemu_pid = pg_util_spawn_qemu(sock_path_graph(branch), sock_read_path_graph(branch), tmp_mac, mac_reader, + 0,0, glob_vm_path, glob_vm_key_path, glob_hugepages_path, errp); diff --git a/tests/vhost/bench-vhost.c b/tests/vhost/bench-vhost.c index eb6296473..f58af7435 100644 --- a/tests/vhost/bench-vhost.c +++ b/tests/vhost/bench-vhost.c @@ -80,6 +80,7 @@ void test_benchmark_vhost(char *vm_image_path, qemu_pid = pg_util_spawn_qemu(socket_path_enter, socket_path_exit, mac1_str, mac2_str, + 0,0, vm_image_path, vm_ssh_key_path, hugepages_path, &error); g_assert(!error); diff --git a/tests/vhost/test-vhost.c b/tests/vhost/test-vhost.c index e9756c50b..921c2aa7c 100644 --- a/tests/vhost/test-vhost.c +++ b/tests/vhost/test-vhost.c @@ -40,7 +40,7 @@ #include "utils/qemu.h" #include "tests.h" -#define NB_PKTS 1 +#define NB_PKTS 10 static uint16_t ssh_port_id = 65000; @@ -103,6 +103,7 @@ static void test_vhost_flow_(int qemu_exit_signal) qemu_pid = pg_util_spawn_qemu(socket_path_0, socket_path_1, mac_addr_0, mac_addr_1, + 0, 0, glob_vm_path, glob_vm_key_path, glob_hugepages_path, &error); @@ -268,6 +269,7 @@ static void test_vhost_multivm_(int qemu_exit_signal) g_assert(socket_path_11); qemu_pid0 = pg_util_spawn_qemu(socket_path_00, socket_path_01, mac_addr_00, mac_addr_01, + 0, 0, glob_vm_path, glob_vm_key_path, glob_hugepages_path, &error); g_assert(qemu_pid0); @@ -286,6 +288,7 @@ static void test_vhost_multivm_(int qemu_exit_signal) qemu_pid1 = pg_util_spawn_qemu(socket_path_10, socket_path_11, mac_addr_10, mac_addr_11, + 0, 0, glob_vm_path, glob_vm_key_path, glob_hugepages_path, &error); g_assert(qemu_pid1); @@ -445,6 +448,7 @@ static void qemu_test(struct qemu_test_params *p) qemu_pid = pg_util_spawn_qemu(p->socket_path[0], p->socket_path[1], mac_0, mac_1, + 0, 0, glob_vm_path, glob_vm_key_path, glob_hugepages_path, &error); @@ -619,6 +623,7 @@ static void test_vhost_destroy(void) qemu_pid = pg_util_spawn_qemu(socket_path_0, socket_path_1, mac_addr_0, mac_addr_1, + 0, 0, glob_vm_path, glob_vm_key_path, glob_hugepages_path, &error); @@ -692,6 +697,7 @@ static void qemu_duo_new(struct qemu_duo_test_params *p) mac = g_strdup_printf("52:54:00:12:34:%02i", i); p->qemu_pid[i] = pg_util_spawn_qemu(socket_path, NULL, mac, NULL, + 0, 0, glob_vm_path, glob_vm_key_path, glob_hugepages_path, @@ -798,8 +804,139 @@ static void test_vhost_seccomp(void) pg_vhost_stop(); } +static void test_vhost_as_cli(void) +{ + int qemu_exit_signal = SIGKILL; + const char mac_addr_0[18] = "52:54:00:12:34:11"; + const char mac_addr_1[18] = "52:54:00:12:34:12"; + struct rte_mempool *mbuf_pool = pg_get_mempool(); + struct pg_brick *vhost_0, *vhost_1, *collect; + struct rte_mbuf *pkts[PG_MAX_PKTS_BURST]; + const char *socket_path_0, *socket_path_1; + struct pg_error *error = NULL; + struct rte_mbuf **result_pkts; + int ret, qemu_pid, i; + uint64_t pkts_mask; + + /* start vhost */ + ret = pg_vhost_start("/tmp", &error); + g_assert(ret == 0); + g_assert(!error); + + /* instanciate brick */ + vhost_0 = pg_vhost_new("vhost-0", PG_VHOST_USER_CLIENT, + &error); + g_assert(!error); + g_assert(vhost_0); + + vhost_1 = pg_vhost_new("vhost-1", PG_VHOST_USER_CLIENT, + &error); + g_assert(!error); + g_assert(vhost_1); + + collect = pg_collect_new("collect", &error); + g_assert(!error); + g_assert(collect); + + /* build the graph */ + pg_brick_link(collect, vhost_1, &error); + g_assert(!error); + + /* spawn first QEMU */ + socket_path_0 = pg_vhost_socket_path(vhost_0); + g_assert(!error); + g_assert(socket_path_0); + socket_path_1 = pg_vhost_socket_path(vhost_1); + g_assert(!error); + g_assert(socket_path_1); + + qemu_pid = pg_util_spawn_qemu(socket_path_0, socket_path_1, + mac_addr_0, mac_addr_1, + 1, 1, + glob_vm_path, + glob_vm_key_path, + glob_hugepages_path, &error); + + g_assert(!error); + g_assert(qemu_pid); + + /* Prepare VM's bridge. */ + SSH("brctl addbr br0", qemu_pid, qemu_exit_signal); + SSH("ifconfig br0 up", qemu_pid, qemu_exit_signal); + SSH("ifconfig ens4 up", qemu_pid, qemu_exit_signal); + SSH("ifconfig ens5 up", qemu_pid, qemu_exit_signal); + SSH("brctl addif br0 ens4", qemu_pid, qemu_exit_signal); + SSH("brctl addif br0 ens5", qemu_pid, qemu_exit_signal); + SSH("brctl setfd br0 0", qemu_pid, qemu_exit_signal); + SSH("brctl stp br0 off", qemu_pid, qemu_exit_signal); + ssh_port_id++; + + /* prepare packet to send */ + for (i = 0; i < NB_PKTS; i++) { + pkts[i] = rte_pktmbuf_alloc(mbuf_pool); + g_assert(pkts[i]); + rte_pktmbuf_append(pkts[i], ETHER_MIN_LEN); + /* set random dst/src mac address so the linux guest bridge + * will not filter them + */ + pg_set_mac_addrs(pkts[i], + "52:54:00:12:34:15", "52:54:00:12:34:16"); + /* set size */ + pg_set_ether_type(pkts[i], ETHER_MIN_LEN - ETHER_HDR_LEN - 4); + } + + /* send packet to the guest via one interface */ + pg_brick_burst_to_east(vhost_0, 0, pkts, + pg_mask_firsts(NB_PKTS), &error); + g_assert(!error); + + /* let the packet propagate and flow */ + for (i = 0; i < 10; i++) { + uint16_t count = 0; + + usleep(100000); + pg_brick_poll(vhost_1, &count, &error); + g_assert(!error); + if (count) + break; + } + + result_pkts = pg_brick_east_burst_get(collect, &pkts_mask, &error); + g_assert(!error); + g_assert(result_pkts); + g_assert(pg_brick_rx_bytes(vhost_0) == 0); + g_assert(pg_brick_tx_bytes(vhost_0) != 0); + g_assert(pg_brick_rx_bytes(vhost_1) != 0); + g_assert(pg_brick_tx_bytes(vhost_1) == 0); + + /* kill QEMU */ + pg_util_stop_qemu(qemu_pid, qemu_exit_signal); + + /* free sent packet */ + for (i = 0; i < NB_PKTS; i++) + rte_pktmbuf_free(pkts[i]); + + /* break the graph */ + pg_brick_unlink(collect, &error); + g_assert(!error); + + /* clean up */ + /* pg_brick_decref(vhost_0, &error); */ + pg_brick_destroy(vhost_0); + g_assert(!error); + pg_brick_destroy(vhost_1); + /* pg_brick_decref(vhost_1, &error); */ + g_assert(!error); + pg_brick_decref(collect, &error); + g_assert(!error); + + /* stop vhost */ + pg_vhost_stop(); +} + void test_vhost(void) { + pg_test_add_func("/vhost/as_cli", test_vhost_as_cli); pg_test_add_func("/vhost/net_classics", test_vhost_net_classics); pg_test_add_func("/vhost/flow", test_vhost_flow); pg_test_add_func("/vhost/multivm", test_vhost_multivm);