Integrate chunk-based OTA updates via desync (OnionUI/Onion#1918)

.github/workflows/tagged-release.yml

@@ -33,7 +33,7 @@ jobs:
       shell: bash
       run: |
         source /root/.bashrc
-        VERSION_OVERRIDE=${{ env.BUILD_VERSION }} make release
+        VERSION_OVERRIDE=${{ env.BUILD_VERSION }} make desync-bin release chunk
     - uses: marvinpinto/action-automatic-releases@latest
       with:
         repo_token: ${{ secrets.GITHUB_TOKEN }}
@@ -42,3 +42,4 @@ jobs:
         title: "Onion V${{ env.BUILD_VERSION }}"
         files: |
           release/Onion-v${{ env.BUILD_VERSION }}.zip
+          release/Onion-v${{ env.BUILD_VERSION }}.caidx

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "third-party/RetroArch-patch"]
 	path = third-party/RetroArch-patch
 	url = https://github.com/Amiga500/RetroArch-patch
+[submodule "third-party/desync"]
+	path = third-party/desync
+	url = https://github.com/folbricht/desync

Makefile

@@ -9,7 +9,7 @@ RA_SUBVERSION=1.22.2-1
 ifneq ($(VERSION_OVERRIDE),)
 VERSION = $(VERSION_OVERRIDE)
 endif
- 
+
 RELEASE_NAME := $(TARGET)-v$(VERSION)
 
 ifdef OS
@@ -43,6 +43,8 @@ PACKAGES_EMU_DEST   := $(PACKAGES_DIR)/Emu
 PACKAGES_APP_DEST   := $(PACKAGES_DIR)/App
 PACKAGES_RAPP_DEST  := $(PACKAGES_DIR)/RApp
 TEMP_DIR            := $(ROOT_DIR)/cache/temp
+DESYNC_SRC          := $(THIRD_PARTY_DIR)/desync
+CHUNK_STORE         := $(RELEASE_DIR)/chunks.castr
 INCLUDE_DIR         := $(ROOT_DIR)/include
 ifeq (,$(GTEST_INCLUDE_DIR))
 GTEST_INCLUDE_DIR = /usr/include/
@@ -63,7 +65,7 @@ APP_COMPONENTS = batteryMonitorUI playActivityUI clock randomGamePicker
 
 ###########################################################
 
-.PHONY: all version core apps external release clean deepclean git-clean with-toolchain patch lib test unit-test \
+.PHONY: all version core apps external release chunk clean deepclean git-clean with-toolchain patch lib test unit-test desync-bin desync-host \
 	$(CORE_COMPONENTS) $(APP_COMPONENTS) installUI
 
 all: dist
@@ -206,8 +208,32 @@ $(THIRD_PARTY_DIR)/RetroArch-patch/bin/retroarch_miyoo354:
 	fi
 	@cd $(THIRD_PARTY_DIR)/RetroArch-patch && make
 
+$(DESYNC_SRC)/go.mod:
+	git submodule update --init third-party/desync
+
+$(BIN_DIR)/desync: $(DESYNC_SRC)/go.mod
+	@$(ECHO) $(COLOR_BLUE)"\n-- Build desync (ARM)"$(COLOR_NORMAL)
+	@cd $(DESYNC_SRC) && GOOS=linux GOARCH=arm GOARM=7 CGO_ENABLED=0 \
+		PATH="$$PATH:/usr/local/go/bin" go build -ldflags="-s -w" -o $(BIN_DIR)/desync ./cmd/desync
+
+desync-bin: $(BIN_DIR)/desync
+
+$(CACHE)/desync: $(DESYNC_SRC)/go.mod
+	@$(ECHO) $(COLOR_BLUE)"\n-- Build desync (host)"$(COLOR_NORMAL)
+	@mkdir -p $(CACHE)
+	@cd $(DESYNC_SRC) && PATH="$$PATH:/usr/local/go/bin" \
+		go build -ldflags="-s -w" -o $(CACHE)/desync ./cmd/desync
+
+desync-host: $(CACHE)/desync
+
 external: $(CACHE)/.setup $(THIRD_PARTY_DIR)/RetroArch-patch/bin/retroarch_miyoo354
 	@$(ECHO) $(PRINT_RECIPE)
+# Add desync to installer (must be pre-built on host with: make desync-bin)
+	@if [ -f $(BIN_DIR)/desync ]; then \
+		cp $(BIN_DIR)/desync $(INSTALLER_DIR)/bin/; \
+	else \
+		echo "WARNING: desync not found — run 'make desync-bin' on the host before building"; \
+	fi
 # Add RetroArch
 	@cp $(THIRD_PARTY_DIR)/RetroArch-patch/bin/* $(BUILD_DIR)/RetroArch/
 	@echo $(RA_SUBVERSION) > $(BUILD_DIR)/RetroArch/onion_ra_version.txt
@@ -242,27 +268,38 @@ dist: build
 # Package configs
 	@cp -R $(TEMP_DIR)/configs/Saves/CurrentProfile/ $(TEMP_DIR)/configs/Saves/GuestProfile
 	@echo -n "Packaging configs..."
-	@cd $(TEMP_DIR)/configs && 7z a -mtm=off $(BUILD_DIR)/.tmp_update/config/configs.pak . -bsp1 -bso0
+	@cd $(TEMP_DIR)/configs && 7z a -ms=off -mtm=off $(BUILD_DIR)/.tmp_update/config/configs.pak . -bsp1 -bso0
 	@echo " DONE"
 	@rm -rf $(TEMP_DIR)/configs
 	@rmdir $(TEMP_DIR)
 # Package RetroArch separately
 	@echo -n "Packaging RetroArch..."
-	@cd $(BUILD_DIR) && 7z a -mtm=off retroarch.pak ./RetroArch -bsp1 -bso0
+	@cd $(BUILD_DIR) && 7z a -ms=off -mtm=off retroarch.pak ./RetroArch -bsp1 -bso0
 	@echo " DONE"
 	@mkdir -p $(DIST_DIR)/RetroArch
 	@mv $(BUILD_DIR)/retroarch.pak $(DIST_DIR)/RetroArch/
 	@echo $(RA_SUBVERSION) > $(DIST_DIR)/RetroArch/ra_package_version.txt
 # Package Onion core
 	@echo -n "Packaging Onion..."
-	@cd $(BUILD_DIR) && 7z a -mtm=off $(DIST_DIR)/miyoo/app/.tmp_update/onion.pak . -x!RetroArch -bsp1 -bso0
+	@cd $(BUILD_DIR) && 7z a -ms=off -mtm=off $(DIST_DIR)/miyoo/app/.tmp_update/onion.pak . -x!RetroArch -bsp1 -bso0
 	@echo " DONE"
 	@$(ECHO) $(PRINT_DONE)
 
 release: dist
 	@$(ECHO) $(PRINT_RECIPE)
 	@rm -f $(RELEASE_DIR)/$(RELEASE_NAME).zip
-	@cd $(DIST_DIR) && 7z a -mtc=off $(RELEASE_DIR)/$(RELEASE_NAME).zip . -bsp1 -bso0
+	@find $(DIST_DIR) -type f -exec touch -t 198001010000 {} +
+	@cd $(DIST_DIR) && 7z a $(RELEASE_DIR)/$(RELEASE_NAME).zip . -bsp1 -bso0
+	@$(ECHO) $(PRINT_DONE)
+
+chunk:
+	@$(ECHO) $(PRINT_RECIPE)
+	@echo "Generating desync index and chunks..."
+	@mkdir -p $(CHUNK_STORE)
+	@$(CACHE)/desync make \
+		-s $(CHUNK_STORE) -m 512:1024:2048 \
+		$(RELEASE_DIR)/$(RELEASE_NAME).caidx \
+		$(RELEASE_DIR)/$(RELEASE_NAME).zip
 	@$(ECHO) $(PRINT_DONE)
 
 clean:
@@ -283,7 +320,7 @@ dev: clean
 
 asan: clean
 	@$(MAKE_ASAN)
-	
+
 git-clean:
 	@git clean -xfd -e .vscode
 

static/build/.tmp_update/script/ota_update.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
 # OTA updates for Onion.
 cmd=$1
+chunk_download=0
+for arg in "$@"; do
+	[ "$arg" = "--chunk" ] && chunk_download=1
+done
 sysdir=/mnt/SDCARD/.tmp_update
 
 # Colors
@@ -13,6 +17,9 @@ NC='\033[0m' # No Color
 # Repository name :
 GITHUB_REPOSITORY=Amiga500/Onion
 
+# Chunk store for desync delta downloads (can be overridden via environment)
+CHUNK_STORE_URL=${CHUNK_STORE_URL:-https://onionui.github.io/Onion/chunks/}
+
 # channel : stable or beta
 channel=$(cat "$sysdir/config/ota_channel" 2> /dev/null)
 if [ "$channel" = "" ]; then
@@ -60,8 +67,15 @@ check_available_space() {
 	mount_point=$(mount | grep -m 1 '/mnt/SDCARD' | awk '{print $1}') # it could be /dev/mmcblk0p1 or /dev/mmcblk0
 	available_space=$(df -m $mount_point | awk 'NR==2{print $4}')
 
-	# Check available space
-	if [ "$available_space" -lt "1000" ]; then
+	# Account for existing seed zip (seed + new download must coexist during update)
+	seed_size=0
+	seed_zip=$(ls "$sysdir/seed/Onion-v"*.zip 2>/dev/null | head -1)
+	[ -n "$seed_zip" ] && seed_size=$(du -m "$seed_zip" | awk '{print $1}')
+
+	# download (~1000MB) + seed on disk + ~300MB extraction headroom
+	required=$((seed_size + 1300))
+
+	if [ "$available_space" -lt "$required" ]; then
 		echo -e "${RED}Available space is insufficient on SD card${NC}\n"
 		echo -ne "${YELLOW}"
 		read -n 1 -s -r -p "Press A to exit"
@@ -138,6 +152,7 @@ get_release_info() {
 	fi
 
 	Release_url=$(echo "$Release_asset" | jq '.browser_download_url' | tr -d '"')
+	Release_caidx_url=$(echo "$Release_assets_info" | jq -r '.assets[]? | select(.name | endswith(".caidx")) | .browser_download_url')
 	Release_FullVersion=$(echo "$Release_asset" | jq '.name' | tr -d "\"" | sed 's/^Onion-v//g' | sed 's/\.zip$//g')
 	Release_Version=$(echo "$Release_FullVersion" | sed 's/-.*$//g')
 	Release_size=$(echo "$Release_asset" | jq -r '.size')
@@ -175,11 +190,42 @@ get_release_info() {
 	return 0
 }
 
+do_download() {
+	seed_zip=$(ls "$sysdir/seed/Onion-v"*.zip 2>/dev/null | head -1)
+	seed_idx=$(ls "$sysdir/seed/Onion-v"*.caidx 2>/dev/null | head -1)
+	desync_bin="$sysdir/bin/desync"
+	out_zip="$sysdir/download/$Release_Version.zip"
+	out_idx="$sysdir/download/$Release_Version.caidx"
+
+	# Always fetch the index (it's small) so it can seed future updates
+	if [ -n "$Release_caidx_url" ]; then
+		wget --no-check-certificate "$Release_caidx_url" -O "$out_idx"
+	fi
+
+	if [ "$chunk_download" -eq 1 ] && [ -f "$out_idx" ] && [ -f "$desync_bin" ] && [ -n "$seed_zip" ] && [ -n "$seed_idx" ]; then
+		echo -ne "\n${BLUE}== Delta download via desync (seed: $(basename $seed_zip)) ==${NC}\n"
+		"$desync_bin" extract \
+			-s "$CHUNK_STORE_URL" \
+			--seed "$seed_idx:$seed_zip" \
+			"$out_idx" \
+			"$out_zip"
+		if [ $? -eq 0 ]; then return 0; fi
+		echo -e "${YELLOW}desync failed — falling back to full download${NC}"
+		rm -f "$out_zip"
+	fi
+
+	echo -ne "\n${BLUE}== Downloading Onion $Release_Version ($channel channel) ==${NC}\n"
+	wget --no-check-certificate "$Release_url" -O "$out_zip"
+}
+
 download_update() {
 	echo -ne "${YELLOW}"
 	read -n 1 -s -r -p "Press A to continue"
 	echo -ne "${NC}"
 
+	# Clean up any stale partial download from a previous failed attempt
+	rm -f "$sysdir/download/"*.zip "$sysdir/download/"*.caidx
+
 	Mychoice=$(echo -e "No\nYes" | $sysdir/script/shellect.sh -t "Download $Release_Version ($Release_size_MB) ?" -b "Press A to validate your choice.")
 	clear
 	if [ "$Mychoice" = "Yes" ]; then
@@ -195,11 +241,9 @@ download_update() {
 		fsck.fat -a $mount_point
 
 		mkdir -p $sysdir/download/
-		echo -ne "\n\n" \
-			"${BLUE}== Downloading Onion $Release_Version ($channel channel) ==${NC}\n"
 		/mnt/SDCARD/.tmp_update/bin/freemma > /dev/null
 		sync
-		wget --no-check-certificate "$Release_url" -O "$sysdir/download/$Release_Version.zip"
+		do_download
 		if [ $? -ne 0 ]; then
 			echo -ne "\n\n" \
 				"${RED}Error: Download failed${NC}\n"
@@ -245,6 +289,16 @@ apply_update() {
 		if [ $? -eq 0 ]; then
 			echo -e "${GREEN}Decompression successful.${NC}"
 			sync
+
+			# Promote downloaded zip and index to seed for next OTA delta update
+			mkdir -p $sysdir/seed
+			rm -f $sysdir/seed/Onion-v*.zip $sysdir/seed/Onion-v*.caidx
+			mv "$sysdir/download/$Release_Version.zip" "$sysdir/seed/$Release_Version.zip"
+			[ -f "$sysdir/download/$Release_Version.caidx" ] && \
+				mv "$sysdir/download/$Release_Version.caidx" "$sysdir/seed/$Release_Version.caidx"
+			echo "$Release_Version" > $sysdir/seed/seed_version.txt
+			sync
+
 			sleep 3
 			echo -ne "\n\n" \
 				"Update $Release_Version applied.\n" \

static/dist/miyoo/app/.tmp_update/install.sh

@@ -375,6 +375,7 @@ run_installation() {
     fi
 
     rm -f $sysdir/config/currentSlide 2> /dev/null
+    promote_initial_seed
     sync
 }
 
@@ -719,6 +720,32 @@ free_mma() {
     /mnt/SDCARD/.tmp_update/bin/freemma
 }
 
+promote_initial_seed() {
+    staged_zip=$(ls "$sysdir/download/Onion-v"*.zip 2>/dev/null | head -1)
+    [ -z "$staged_zip" ] && return 0
+
+    staged_idx=$(ls "$sysdir/download/Onion-v"*.caidx 2>/dev/null | head -1)
+
+    echo ":: Promote initial seed: $(basename $staged_zip)"
+    mkdir -p "$sysdir/seed"
+    rm -f "$sysdir/seed"/Onion-v*.zip "$sysdir/seed"/Onion-v*.caidx
+
+    seed_zip="$sysdir/seed/$(basename $staged_zip)"
+    seed_idx="${seed_zip%.zip}.caidx"
+
+    mv "$staged_zip" "$seed_zip"
+
+    if [ -n "$staged_idx" ]; then
+        mv "$staged_idx" "$seed_idx"
+    elif [ -f "$sysdir/bin/desync" ]; then
+        echo ":: Generating seed index (no .caidx found)..."
+        "$sysdir/bin/desync" make "$seed_idx" "$seed_zip"
+    fi
+
+    echo "$(basename $staged_zip .zip)" > "$sysdir/seed/seed_version.txt"
+    sync
+}
+
 main
 sync
 reboot