Fix flaky/hanging note-c card.binary HIL tests

.github/workflows/notecard-binary-tests.yml

@@ -19,6 +19,14 @@ permissions:
   contents: read
   id-token: write
 
+# Don't let a run collide with itself: a new push to the same ref (e.g. a PR
+# update) cancels the in-flight run instead of racing it for the single
+# barcelona-notestation-1 reservation. Different refs (other PRs, the nightly
+# schedule) get separate groups and are unaffected.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   md5srv-test:
     uses: ./.github/workflows/md5srv-tests.yml
@@ -30,6 +38,10 @@ jobs:
       github.event.repository.fork != true &&
       (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
     runs-on: ubuntu-latest
+    # A healthy run reserves, flashes, and runs the suite in well under this.
+    # The cap ensures a stuck serial monitor fails fast instead of burning the
+    # default 6h job timeout.
+    timeout-minutes: 30
     defaults:
       run:
         shell: bash
@@ -69,7 +81,7 @@ jobs:
           aws-region: us-east-1
           mask-aws-account-id: 'false'
 
-      # Needed for asyncio.TaskGroup, which the card_client code uses.
+      # Needed to run PlatformIO (installed and invoked below).
       - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
@@ -98,18 +110,6 @@ jobs:
           sudo apt update
           sudo apt install gdb-multiarch ngrok jq
 
-      # We need socat 1.7.4.4 because prior to that there are some issues with
-      # setting the baud rate properly. socat is used by the card_client code
-      # for various tunnels (e.g. Swan USB port, Swan OpenOCD server, etc.).
-      - name: Install socat 1.7.4.4
-        run: |
-          wget http://www.dest-unreach.org/socat/download/socat-1.7.4.4.tar.gz
-          tar xvf socat-1.7.4.4.tar.gz
-          cd socat-1.7.4.4
-          ./configure
-          make -j
-          sudo make install
-
       - name: Install PlatformIO dependencies
         run: |
           python -m pip install --upgrade pip
@@ -193,49 +193,28 @@ jobs:
             exit 1
           fi
 
-      # The notestation CLI uses the notecard CLI.
-      - name: Install notecard CLI
-        run: |
-          git clone --depth=1 https://github.com/blues/note-cli.git
-          cd note-cli/notecard
-          go build .
-          sudo cp notecard /usr/bin/
-
-      - name: Download and install notestation CLI v2.2.0
-        env:
-          GH_TOKEN: ${{ secrets.NOTESTATION_RELEASE_DOWNLOAD_TOKEN }}
-        run: |
-          api_url="https://api.github.com/repos/blues/notestation/releases/tags/v2.2.0"
-
-          # Query the release asset metadata.
-          asset_info=$(curl -s -H "Authorization: token $GH_TOKEN" "$api_url")
-
-          # Extract the release asset URL and name.
-          asset_url=$(echo "$asset_info" | jq -r '.assets[] | select(.name | endswith(".whl")) | .url')
-          asset_name=$(echo "$asset_info" | jq -r '.assets[] | select(.name | endswith(".whl")) | .name')
-
-          if [ -z "$asset_url" ] || [ -z "$asset_name" ]; then
-            echo "ERROR: No wheel found in release assets."
-            exit 1
-          fi
-
-          # Download the wheel using the original filename.
-          curl -L -H "Authorization: token $GH_TOKEN" \
-               -H "Accept: application/octet-stream" \
-               "$asset_url" -o "$asset_name"
-
-          # Create a virtual environment and install the notestation CLI in it.
-          python -m venv venv
-          source venv/bin/activate
-          wheel_file=$(ls *.whl)
-          pip install "$wheel_file"
+      # Use the Go notestation-client (notestation-v3.2.1+). Its SerialTunnel
+      # publishes the host_mcu_usb symlink only after the server confirms the
+      # USB device is live, and removes it on drop -- so the symlink is a
+      # reliable device-liveness signal, which is what after_upload.py checks
+      # with os.path.exists() before running the tests. The old v2.2.0 Python
+      # client built the tunnel with `socat PTY,link=host_mcu_usb TCP:...`,
+      # which created the symlink as soon as socat started -- before the Swan's
+      # USB was up -- so it flickered while the Swan re-enumerated after
+      # flashing: after_upload.py caught a blink and let the run proceed, then
+      # socat removed the symlink before PlatformIO opened the port, giving
+      # "could not open port .../host_mcu_usb" and 0 tests. See
+      # blues/notestation f6fc011 (first released in notestation-v3.2.1).
+      - name: Install notestation-client
+        uses: blues/notestation-actions/install_notestation_client@v1.0.2
+        with:
+          version: notestation-v3.5.1
+          token: ${{ secrets.NOTESTATION_RELEASE_DOWNLOAD_TOKEN }}
 
       - name: Build and upload test firmware and run tests
         run: |
-          source venv/bin/activate
-
           # Reserve the Notestation in the background
-          nohup notestation client reserve --notestation barcelona-notestation-1 &> card_client.log &
+          nohup notestation-client reserve --notestation barcelona-notestation-1 &> card_client.log &
           PID=$!
 
           timeout=600  # 10 minutes in seconds
@@ -265,10 +244,10 @@ jobs:
 
           echo "Notestation reserved."
 
-          # If firmware update is requested, flash it using notestation client flash
+          # If firmware update is requested, flash it using notestation-client flash
           if [ "${{ env.FIRMWARE_S3_URI }}" == "nightly" ]; then
             echo "Flashing nightly firmware..."
-            notestation client flash --notecard-nightly
+            notestation-client flash --notecard-nightly --notestation barcelona-notestation-1
             echo "Firmware flash complete."
           fi
 
@@ -299,8 +278,10 @@ jobs:
           # Enable Notecard USB trace output.
           printf '{"req":"card.trace","mode":"usb","trace":"req,comm,sync,mem"}\n' > "$RESERVATION_DIR/notecard_usb"
 
-          # Stream Notecard USB output to trace.log in the background.
-          socat -u GOPEN:"$RESERVATION_DIR/notecard_usb",rawer - >> "$GITHUB_WORKSPACE/trace.log" &
+          # Stream Notecard USB output to trace.log in the background. The
+          # notestation-client already exposes notecard_usb as a raw PTY, so a
+          # plain read is enough -- no socat needed.
+          cat "$RESERVATION_DIR/notecard_usb" >> "$GITHUB_WORKSPACE/trace.log" &
           TRACE_PID=$!
           echo "Notecard trace capture started (PID: $TRACE_PID)."
 

test/hitl/card.binary/after_upload.py

@@ -2,20 +2,21 @@
 # runs the after_upload method registered in this file.
 
 import os
+import select
 import time
 
 Import("env")
 
 
 # There's an issue with PlatformIO where, after uploading, it expects to see the
 # test port (i.e. the Swan's USB, which outputs test logging) too quickly. After
-# upload, it may take a second for the device to appear, and so we don't want to
+# upload, it may take a moment for the device to appear, and so we don't want to
 # proceed past the upload phase until it's appeared.
 def _wait_for_test_port(env):
     port = env.GetProjectOption("test_port")
     print("Waiting for test port: " + port)
 
-    timeout_secs = 10
+    timeout_secs = 30
     start = time.time()
     while not os.path.exists(port):
         if (time.time() - start) > timeout_secs:
@@ -25,12 +26,73 @@ def _wait_for_test_port(env):
     return port
 
 
-# The test firmware doesn't kick off until it receives the string "start\n" or
-# "start\r\n" on the Swan's USB port.
+# The test firmware (test/test_main.cpp) does not run the tests until it receives
+# "start\n" on the Swan's USB. It only starts listening once it has booted far
+# enough to reach its read loop, at which point it prints:
+#
+#     Waiting to receive "start"...
+#
+# The freshly-reset Swan and the test port can become ready at nearly the same
+# instant, so sending "start" the moment the port appears races the firmware's
+# boot: the byte can be dropped before the read loop is running, leaving the
+# firmware waiting forever with no output (a silent hang of the test step).
+#
+# To avoid that race, wait until we have actually seen the firmware's prompt,
+# then send "start" -- at that point the firmware is guaranteed to be in its
+# read loop. If the prompt isn't seen within the timeout (e.g. it was emitted
+# before we opened the port and wasn't buffered), fall back to sending anyway,
+# which is no worse than the previous unconditional behaviour.
+#
+# Use plain os-level file I/O rather than pyserial: opening via pyserial toggles
+# the DTR/RTS modem-control lines, which errors on this virtual (TCP-backed)
+# serial device -- the same reason the tests run with --no-reset.
 def _start_tests(port):
-    print("Sending 'start\\n' to Swan to kick off tests...")
-    with open(port, "wb", buffering=0) as f:
-        f.write(b"start\n")
+    prompt = b'Waiting to receive "start"'
+    prompt_timeout = 20  # seconds to wait for the firmware's prompt
+
+    fd = os.open(port, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
+    try:
+        # 1) Wait until the firmware prints its prompt, i.e. it has booted into
+        #    its read loop and is listening.
+        buf = b""
+        deadline = time.time() + prompt_timeout
+        saw_prompt = False
+        while time.time() < deadline:
+            r, _, _ = select.select([fd], [], [], 0.5)
+            if not r:
+                continue
+            try:
+                data = os.read(fd, 4096)
+            except (BlockingIOError, OSError):
+                continue
+            if data:
+                buf = (buf + data)[-4096:]  # the prompt is short; keep the tail
+                if prompt in buf:
+                    saw_prompt = True
+                    break
+        if saw_prompt:
+            print('Saw prompt; sending "start" to kick off tests...')
+        else:
+            print(
+                f"Did not see the firmware prompt within {prompt_timeout}s; "
+                'sending "start" anyway.'
+            )
+
+        # 2) Send "start" as a short burst rather than once. The very first send
+        #    can be dropped while the just-reset Swan's USB re-enumeration
+        #    settles (a ~2s window observed on hardware); a repeat a couple
+        #    seconds later lands. Extra "start"s after the firmware has begun
+        #    are harmless -- it has already left its read loop. Kept short (a few
+        #    seconds) so it finishes before the firmware's post-start delay
+        #    elapses and the test-monitor step takes over.
+        for _ in range(8):
+            try:
+                os.write(fd, b"start\n")
+            except OSError:
+                pass
+            time.sleep(0.4)
+    finally:
+        os.close(fd)
 
 
 def after_upload(source, target, env):