From c34e44e324bc37748572cfcb85fc27ddc8bae9d7 Mon Sep 17 00:00:00 2001
From: Vika Shleina <vsh@nyantec.com>
Date: Fri, 13 Dec 2024 23:55:43 +0300
Subject: [PATCH 1/2] WIP: qemu: randomly choose vsock port on the host side

Prevents conflicts between MicroVMs.

Somehow doesn't work first try; VM might timeout a few times before it starts
working. The problem is most likely between socat and systemd. More debug
logging is needed. (Adding it here would be trivial, actually.)
---
 lib/runners/qemu.nix | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/runners/qemu.nix b/lib/runners/qemu.nix
index cc9c27f8..56f7301a 100644
--- a/lib/runners/qemu.nix
+++ b/lib/runners/qemu.nix
@@ -160,10 +160,18 @@ lib.warnIf (mem == 2048) ''
   preStart = ''
     ${microvmConfig.preStart}
   '' + lib.optionalString supportsNotifySocket ''
+    # Clean up temporary files.
+    ${pkgs.coreutils}/bin/rm -f ./socat.log ./notify_socket.cred
+    # Default value: for running without systemd.
+    NOTIFY_VSOCK_PORT=8888
     # Start socat to forward systemd notify socket over vsock
     if [ -n "''${NOTIFY_SOCKET-}" ]; then
-      ${pkgs.socat}/bin/socat VSOCK-LISTEN:8888,fork UNIX-SENDTO:$NOTIFY_SOCKET &
+        ${pkgs.socat}/bin/socat -d -d VSOCK-LISTEN:-1,fork UNIX-SENDTO:$NOTIFY_SOCKET 2> ./socat.log &
+        NOTIFY_VSOCK_PORT=$({ ${pkgs.coreutils}/bin/tail -f ./socat.log || true; } | ${pkgs.gawk}/bin/awk '/listening on/ { if (match($0, /port:[0-9]*/)) { print substr($0, RSTART + 5, RLENGTH - 5); exit; } }')
+        echo "Picked port for notify vsock: $NOTIFY_VSOCK_PORT"
     fi
+    echo -n "vsock-stream:2:$NOTIFY_VSOCK_PORT" > ./notify_socket.cred
+
   '';
 
   command = lib.escapeShellArgs (
@@ -314,7 +322,7 @@ lib.warnIf (mem == 2048) ''
       # tried, SMBIOS Type 11 entries simply don't work. It looks like it might
       # be broken on QEMU side. Why? I don't know.
       "-fw_cfg"
-      "name=opt/io.systemd.credentials/vmm.notify_socket,string=vsock-stream:2:8888"
+      "name=opt/io.systemd.credentials/vmm.notify_socket,file=./notify_socket.cred"
     ]
     ++
     extraArgs

From d364fdcad2e896973d05d5ea4f3aed059ba16b76 Mon Sep 17 00:00:00 2001
From: Vika Shleina <vsh@nyantec.com>
Date: Tue, 24 Dec 2024 21:31:51 +0300
Subject: [PATCH 2/2] QEMU: use a static offset to choose vsock port on the
 host side

600000 is arbitrarily picked to avoid conflicts with the "standard" networking
range of 65535 ports, since vsock supports 32-bit port numbers.
---
 lib/runners/qemu.nix | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/lib/runners/qemu.nix b/lib/runners/qemu.nix
index 56f7301a..b01babcd 100644
--- a/lib/runners/qemu.nix
+++ b/lib/runners/qemu.nix
@@ -147,6 +147,7 @@ let
     else "";
 
   supportsNotifySocket = vsock.cid != null;
+  vsockHostPortOffset = 600000;
 
 in
 lib.warnIf (mem == 2048) ''
@@ -163,15 +164,11 @@ lib.warnIf (mem == 2048) ''
     # Clean up temporary files.
     ${pkgs.coreutils}/bin/rm -f ./socat.log ./notify_socket.cred
     # Default value: for running without systemd.
-    NOTIFY_VSOCK_PORT=8888
+    NOTIFY_VSOCK_PORT=${toString (vsockHostPortOffset + vsock.cid)}
     # Start socat to forward systemd notify socket over vsock
     if [ -n "''${NOTIFY_SOCKET-}" ]; then
-        ${pkgs.socat}/bin/socat -d -d VSOCK-LISTEN:-1,fork UNIX-SENDTO:$NOTIFY_SOCKET 2> ./socat.log &
-        NOTIFY_VSOCK_PORT=$({ ${pkgs.coreutils}/bin/tail -f ./socat.log || true; } | ${pkgs.gawk}/bin/awk '/listening on/ { if (match($0, /port:[0-9]*/)) { print substr($0, RSTART + 5, RLENGTH - 5); exit; } }')
-        echo "Picked port for notify vsock: $NOTIFY_VSOCK_PORT"
+        ${pkgs.socat}/bin/socat -d -d VSOCK-LISTEN:$NOTIFY_VSOCK_PORT,fork UNIX-SENDTO:$NOTIFY_SOCKET 2> ./socat.log &
     fi
-    echo -n "vsock-stream:2:$NOTIFY_VSOCK_PORT" > ./notify_socket.cred
-
   '';
 
   command = lib.escapeShellArgs (
@@ -322,7 +319,7 @@ lib.warnIf (mem == 2048) ''
       # tried, SMBIOS Type 11 entries simply don't work. It looks like it might
       # be broken on QEMU side. Why? I don't know.
       "-fw_cfg"
-      "name=opt/io.systemd.credentials/vmm.notify_socket,file=./notify_socket.cred"
+      "name=opt/io.systemd.credentials/vmm.notify_socket,string=vsock-stream:2:${toString (vsockHostPortOffset + vsock.cid)}"
     ]
     ++
     extraArgs