From a6ffff3db1c19ddf4ae3b5b050d3545e57039381 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 18 Mar 2026 14:38:00 +0000 Subject: [PATCH 1/2] Initial plan From 35b9c8a5aac0ca8d64211b938e24804293e295a2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 18 Mar 2026 14:42:57 +0000 Subject: [PATCH 2/2] Integrate PR OnionUI/Onion#1918: Enable chunk-based OTA updates via desync Add desync submodule for content-addressable chunking. Update Makefile with desync build targets, 7z deterministic archive flags, and chunk generation target. Update OTA update script with delta download support via --chunk flag. Update install.sh with promote_initial_seed function. Update tagged-release workflow to build desync and include .caidx artifact. Co-authored-by: Amiga500 <16525337+Amiga500@users.noreply.github.com> --- .github/workflows/tagged-release.yml | 3 +- .gitmodules | 3 + Makefile | 51 +++++++++++++-- static/build/.tmp_update/script/ota_update.sh | 64 +++++++++++++++++-- static/dist/miyoo/app/.tmp_update/install.sh | 27 ++++++++ third-party/desync | 1 + 6 files changed, 136 insertions(+), 13 deletions(-) create mode 160000 third-party/desync diff --git a/.github/workflows/tagged-release.yml b/.github/workflows/tagged-release.yml index cd8548df8e..f058ba9b8e 100644 --- a/.github/workflows/tagged-release.yml +++ b/.github/workflows/tagged-release.yml @@ -33,7 +33,7 @@ jobs: shell: bash run: | source /root/.bashrc - VERSION_OVERRIDE=${{ env.BUILD_VERSION }} make release + VERSION_OVERRIDE=${{ env.BUILD_VERSION }} make desync-bin release chunk - uses: marvinpinto/action-automatic-releases@latest with: repo_token: ${{ secrets.GITHUB_TOKEN }} @@ -42,3 +42,4 @@ jobs: title: "Onion V${{ env.BUILD_VERSION }}" files: | release/Onion-v${{ env.BUILD_VERSION }}.zip + release/Onion-v${{ env.BUILD_VERSION }}.caidx diff --git a/.gitmodules b/.gitmodules index d05c297bcc..58d7711700 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "third-party/RetroArch-patch"] path = third-party/RetroArch-patch url = https://github.com/Amiga500/RetroArch-patch +[submodule "third-party/desync"] + path = third-party/desync + url = https://github.com/folbricht/desync diff --git a/Makefile b/Makefile index b3158e1941..c2f8db3b0d 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ RA_SUBVERSION=1.22.2-1 ifneq ($(VERSION_OVERRIDE),) VERSION = $(VERSION_OVERRIDE) endif - + RELEASE_NAME := $(TARGET)-v$(VERSION) ifdef OS @@ -43,6 +43,8 @@ PACKAGES_EMU_DEST := $(PACKAGES_DIR)/Emu PACKAGES_APP_DEST := $(PACKAGES_DIR)/App PACKAGES_RAPP_DEST := $(PACKAGES_DIR)/RApp TEMP_DIR := $(ROOT_DIR)/cache/temp +DESYNC_SRC := $(THIRD_PARTY_DIR)/desync +CHUNK_STORE := $(RELEASE_DIR)/chunks.castr INCLUDE_DIR := $(ROOT_DIR)/include ifeq (,$(GTEST_INCLUDE_DIR)) GTEST_INCLUDE_DIR = /usr/include/ @@ -63,7 +65,7 @@ APP_COMPONENTS = batteryMonitorUI playActivityUI clock randomGamePicker ########################################################### -.PHONY: all version core apps external release clean deepclean git-clean with-toolchain patch lib test unit-test \ +.PHONY: all version core apps external release chunk clean deepclean git-clean with-toolchain patch lib test unit-test desync-bin desync-host \ $(CORE_COMPONENTS) $(APP_COMPONENTS) installUI all: dist @@ -206,8 +208,32 @@ $(THIRD_PARTY_DIR)/RetroArch-patch/bin/retroarch_miyoo354: fi @cd $(THIRD_PARTY_DIR)/RetroArch-patch && make +$(DESYNC_SRC)/go.mod: + git submodule update --init third-party/desync + +$(BIN_DIR)/desync: $(DESYNC_SRC)/go.mod + @$(ECHO) $(COLOR_BLUE)"\n-- Build desync (ARM)"$(COLOR_NORMAL) + @cd $(DESYNC_SRC) && GOOS=linux GOARCH=arm GOARM=7 CGO_ENABLED=0 \ + PATH="$$PATH:/usr/local/go/bin" go build -ldflags="-s -w" -o $(BIN_DIR)/desync ./cmd/desync + +desync-bin: $(BIN_DIR)/desync + +$(CACHE)/desync: $(DESYNC_SRC)/go.mod + @$(ECHO) $(COLOR_BLUE)"\n-- Build desync (host)"$(COLOR_NORMAL) + @mkdir -p $(CACHE) + @cd $(DESYNC_SRC) && PATH="$$PATH:/usr/local/go/bin" \ + go build -ldflags="-s -w" -o $(CACHE)/desync ./cmd/desync + +desync-host: $(CACHE)/desync + external: $(CACHE)/.setup $(THIRD_PARTY_DIR)/RetroArch-patch/bin/retroarch_miyoo354 @$(ECHO) $(PRINT_RECIPE) +# Add desync to installer (must be pre-built on host with: make desync-bin) + @if [ -f $(BIN_DIR)/desync ]; then \ + cp $(BIN_DIR)/desync $(INSTALLER_DIR)/bin/; \ + else \ + echo "WARNING: desync not found — run 'make desync-bin' on the host before building"; \ + fi # Add RetroArch @cp $(THIRD_PARTY_DIR)/RetroArch-patch/bin/* $(BUILD_DIR)/RetroArch/ @echo $(RA_SUBVERSION) > $(BUILD_DIR)/RetroArch/onion_ra_version.txt @@ -242,27 +268,38 @@ dist: build # Package configs @cp -R $(TEMP_DIR)/configs/Saves/CurrentProfile/ $(TEMP_DIR)/configs/Saves/GuestProfile @echo -n "Packaging configs..." - @cd $(TEMP_DIR)/configs && 7z a -mtm=off $(BUILD_DIR)/.tmp_update/config/configs.pak . -bsp1 -bso0 + @cd $(TEMP_DIR)/configs && 7z a -ms=off -mtm=off $(BUILD_DIR)/.tmp_update/config/configs.pak . -bsp1 -bso0 @echo " DONE" @rm -rf $(TEMP_DIR)/configs @rmdir $(TEMP_DIR) # Package RetroArch separately @echo -n "Packaging RetroArch..." - @cd $(BUILD_DIR) && 7z a -mtm=off retroarch.pak ./RetroArch -bsp1 -bso0 + @cd $(BUILD_DIR) && 7z a -ms=off -mtm=off retroarch.pak ./RetroArch -bsp1 -bso0 @echo " DONE" @mkdir -p $(DIST_DIR)/RetroArch @mv $(BUILD_DIR)/retroarch.pak $(DIST_DIR)/RetroArch/ @echo $(RA_SUBVERSION) > $(DIST_DIR)/RetroArch/ra_package_version.txt # Package Onion core @echo -n "Packaging Onion..." - @cd $(BUILD_DIR) && 7z a -mtm=off $(DIST_DIR)/miyoo/app/.tmp_update/onion.pak . -x!RetroArch -bsp1 -bso0 + @cd $(BUILD_DIR) && 7z a -ms=off -mtm=off $(DIST_DIR)/miyoo/app/.tmp_update/onion.pak . -x!RetroArch -bsp1 -bso0 @echo " DONE" @$(ECHO) $(PRINT_DONE) release: dist @$(ECHO) $(PRINT_RECIPE) @rm -f $(RELEASE_DIR)/$(RELEASE_NAME).zip - @cd $(DIST_DIR) && 7z a -mtc=off $(RELEASE_DIR)/$(RELEASE_NAME).zip . -bsp1 -bso0 + @find $(DIST_DIR) -type f -exec touch -t 198001010000 {} + + @cd $(DIST_DIR) && 7z a $(RELEASE_DIR)/$(RELEASE_NAME).zip . -bsp1 -bso0 + @$(ECHO) $(PRINT_DONE) + +chunk: + @$(ECHO) $(PRINT_RECIPE) + @echo "Generating desync index and chunks..." + @mkdir -p $(CHUNK_STORE) + @$(CACHE)/desync make \ + -s $(CHUNK_STORE) -m 512:1024:2048 \ + $(RELEASE_DIR)/$(RELEASE_NAME).caidx \ + $(RELEASE_DIR)/$(RELEASE_NAME).zip @$(ECHO) $(PRINT_DONE) clean: @@ -283,7 +320,7 @@ dev: clean asan: clean @$(MAKE_ASAN) - + git-clean: @git clean -xfd -e .vscode diff --git a/static/build/.tmp_update/script/ota_update.sh b/static/build/.tmp_update/script/ota_update.sh index 3a5e153381..a3f4d67606 100644 --- a/static/build/.tmp_update/script/ota_update.sh +++ b/static/build/.tmp_update/script/ota_update.sh @@ -1,6 +1,10 @@ #!/bin/sh # OTA updates for Onion. cmd=$1 +chunk_download=0 +for arg in "$@"; do + [ "$arg" = "--chunk" ] && chunk_download=1 +done sysdir=/mnt/SDCARD/.tmp_update # Colors @@ -13,6 +17,9 @@ NC='\033[0m' # No Color # Repository name : GITHUB_REPOSITORY=Amiga500/Onion +# Chunk store for desync delta downloads (can be overridden via environment) +CHUNK_STORE_URL=${CHUNK_STORE_URL:-https://onionui.github.io/Onion/chunks/} + # channel : stable or beta channel=$(cat "$sysdir/config/ota_channel" 2> /dev/null) if [ "$channel" = "" ]; then @@ -60,8 +67,15 @@ check_available_space() { mount_point=$(mount | grep -m 1 '/mnt/SDCARD' | awk '{print $1}') # it could be /dev/mmcblk0p1 or /dev/mmcblk0 available_space=$(df -m $mount_point | awk 'NR==2{print $4}') - # Check available space - if [ "$available_space" -lt "1000" ]; then + # Account for existing seed zip (seed + new download must coexist during update) + seed_size=0 + seed_zip=$(ls "$sysdir/seed/Onion-v"*.zip 2>/dev/null | head -1) + [ -n "$seed_zip" ] && seed_size=$(du -m "$seed_zip" | awk '{print $1}') + + # download (~1000MB) + seed on disk + ~300MB extraction headroom + required=$((seed_size + 1300)) + + if [ "$available_space" -lt "$required" ]; then echo -e "${RED}Available space is insufficient on SD card${NC}\n" echo -ne "${YELLOW}" read -n 1 -s -r -p "Press A to exit" @@ -138,6 +152,7 @@ get_release_info() { fi Release_url=$(echo "$Release_asset" | jq '.browser_download_url' | tr -d '"') + Release_caidx_url=$(echo "$Release_assets_info" | jq -r '.assets[]? | select(.name | endswith(".caidx")) | .browser_download_url') Release_FullVersion=$(echo "$Release_asset" | jq '.name' | tr -d "\"" | sed 's/^Onion-v//g' | sed 's/\.zip$//g') Release_Version=$(echo "$Release_FullVersion" | sed 's/-.*$//g') Release_size=$(echo "$Release_asset" | jq -r '.size') @@ -175,11 +190,42 @@ get_release_info() { return 0 } +do_download() { + seed_zip=$(ls "$sysdir/seed/Onion-v"*.zip 2>/dev/null | head -1) + seed_idx=$(ls "$sysdir/seed/Onion-v"*.caidx 2>/dev/null | head -1) + desync_bin="$sysdir/bin/desync" + out_zip="$sysdir/download/$Release_Version.zip" + out_idx="$sysdir/download/$Release_Version.caidx" + + # Always fetch the index (it's small) so it can seed future updates + if [ -n "$Release_caidx_url" ]; then + wget --no-check-certificate "$Release_caidx_url" -O "$out_idx" + fi + + if [ "$chunk_download" -eq 1 ] && [ -f "$out_idx" ] && [ -f "$desync_bin" ] && [ -n "$seed_zip" ] && [ -n "$seed_idx" ]; then + echo -ne "\n${BLUE}== Delta download via desync (seed: $(basename $seed_zip)) ==${NC}\n" + "$desync_bin" extract \ + -s "$CHUNK_STORE_URL" \ + --seed "$seed_idx:$seed_zip" \ + "$out_idx" \ + "$out_zip" + if [ $? -eq 0 ]; then return 0; fi + echo -e "${YELLOW}desync failed — falling back to full download${NC}" + rm -f "$out_zip" + fi + + echo -ne "\n${BLUE}== Downloading Onion $Release_Version ($channel channel) ==${NC}\n" + wget --no-check-certificate "$Release_url" -O "$out_zip" +} + download_update() { echo -ne "${YELLOW}" read -n 1 -s -r -p "Press A to continue" echo -ne "${NC}" + # Clean up any stale partial download from a previous failed attempt + rm -f "$sysdir/download/"*.zip "$sysdir/download/"*.caidx + Mychoice=$(echo -e "No\nYes" | $sysdir/script/shellect.sh -t "Download $Release_Version ($Release_size_MB) ?" -b "Press A to validate your choice.") clear if [ "$Mychoice" = "Yes" ]; then @@ -195,11 +241,9 @@ download_update() { fsck.fat -a $mount_point mkdir -p $sysdir/download/ - echo -ne "\n\n" \ - "${BLUE}== Downloading Onion $Release_Version ($channel channel) ==${NC}\n" /mnt/SDCARD/.tmp_update/bin/freemma > /dev/null sync - wget --no-check-certificate "$Release_url" -O "$sysdir/download/$Release_Version.zip" + do_download if [ $? -ne 0 ]; then echo -ne "\n\n" \ "${RED}Error: Download failed${NC}\n" @@ -245,6 +289,16 @@ apply_update() { if [ $? -eq 0 ]; then echo -e "${GREEN}Decompression successful.${NC}" sync + + # Promote downloaded zip and index to seed for next OTA delta update + mkdir -p $sysdir/seed + rm -f $sysdir/seed/Onion-v*.zip $sysdir/seed/Onion-v*.caidx + mv "$sysdir/download/$Release_Version.zip" "$sysdir/seed/$Release_Version.zip" + [ -f "$sysdir/download/$Release_Version.caidx" ] && \ + mv "$sysdir/download/$Release_Version.caidx" "$sysdir/seed/$Release_Version.caidx" + echo "$Release_Version" > $sysdir/seed/seed_version.txt + sync + sleep 3 echo -ne "\n\n" \ "Update $Release_Version applied.\n" \ diff --git a/static/dist/miyoo/app/.tmp_update/install.sh b/static/dist/miyoo/app/.tmp_update/install.sh index a70143fd53..ef3558750f 100644 --- a/static/dist/miyoo/app/.tmp_update/install.sh +++ b/static/dist/miyoo/app/.tmp_update/install.sh @@ -375,6 +375,7 @@ run_installation() { fi rm -f $sysdir/config/currentSlide 2> /dev/null + promote_initial_seed sync } @@ -719,6 +720,32 @@ free_mma() { /mnt/SDCARD/.tmp_update/bin/freemma } +promote_initial_seed() { + staged_zip=$(ls "$sysdir/download/Onion-v"*.zip 2>/dev/null | head -1) + [ -z "$staged_zip" ] && return 0 + + staged_idx=$(ls "$sysdir/download/Onion-v"*.caidx 2>/dev/null | head -1) + + echo ":: Promote initial seed: $(basename $staged_zip)" + mkdir -p "$sysdir/seed" + rm -f "$sysdir/seed"/Onion-v*.zip "$sysdir/seed"/Onion-v*.caidx + + seed_zip="$sysdir/seed/$(basename $staged_zip)" + seed_idx="${seed_zip%.zip}.caidx" + + mv "$staged_zip" "$seed_zip" + + if [ -n "$staged_idx" ]; then + mv "$staged_idx" "$seed_idx" + elif [ -f "$sysdir/bin/desync" ]; then + echo ":: Generating seed index (no .caidx found)..." + "$sysdir/bin/desync" make "$seed_idx" "$seed_zip" + fi + + echo "$(basename $staged_zip .zip)" > "$sysdir/seed/seed_version.txt" + sync +} + main sync reboot diff --git a/third-party/desync b/third-party/desync new file mode 160000 index 0000000000..e58d685523 --- /dev/null +++ b/third-party/desync @@ -0,0 +1 @@ +Subproject commit e58d685523ab46d7084d97dcbc7a5c661d73d563