From f6a348a59637d4e227b1b14af8f25a56ef7ba6fa Mon Sep 17 00:00:00 2001 From: Howard20181 <40033067+Howard20181@users.noreply.github.com> Date: Fri, 26 Aug 2022 02:33:27 +0800 Subject: [PATCH] Download on demand Delete only when files are corrupted Download only when a new version of a file is available close #39 --- scripts/extractWSA.py | 5 +- scripts/generateGappsLink.py | 14 ++--- scripts/generateMagiskLink.py | 27 +++++----- scripts/generateWSALinks.py | 21 ++++---- scripts/run.sh | 98 ++++++++++++++++++++++++----------- 5 files changed, 98 insertions(+), 67 deletions(-) diff --git a/scripts/extractWSA.py b/scripts/extractWSA.py index 2c2b10c..9da9204 100644 --- a/scripts/extractWSA.py +++ b/scripts/extractWSA.py @@ -32,8 +32,9 @@ arch = sys.argv[1] if not os.path.exists(Path.cwd().parent / sys.argv[2] / "wsa"): os.makedirs(Path.cwd().parent / sys.argv[2] / "wsa") zip_name = "" -workdir = Path.cwd().parent / sys.argv[2] / "wsa" -with zipfile.ZipFile(Path.cwd().parent / "download/wsa.zip") as zip: +wsa_zip_path= Path(sys.argv[2]).resolve() +workdir = Path.cwd().parent / sys.argv[3] / "wsa" +with zipfile.ZipFile(wsa_zip_path) as zip: for f in zip.filelist: if arch in f.filename.lower(): zip_name = f.filename diff --git a/scripts/generateGappsLink.py b/scripts/generateGappsLink.py index 0962334..06cc559 100644 --- a/scripts/generateGappsLink.py +++ b/scripts/generateGappsLink.py @@ -50,12 +50,8 @@ except Exception: print(f"download link: {link}", flush=True) -out_file = download_dir / "gapps.zip" - -if not os.path.isfile(out_file): - # urllib.request.urlretrieve(link, out_file) - with open(download_dir/tempScript, 'a') as f: - f.writelines(f'{link}\n') - f.writelines(f' dir={download_dir}\n') - f.writelines(f' out=gapps.zip\n') - f.close +with open(download_dir/tempScript, 'a') as f: + f.writelines(f'{link}\n') + f.writelines(f' dir={download_dir}\n') + f.writelines(f' out=OpenGApps-{arch}-{variant}.zip\n') + f.close diff --git a/scripts/generateMagiskLink.py b/scripts/generateMagiskLink.py index 6c541ce..0578c4b 100644 --- a/scripts/generateMagiskLink.py +++ b/scripts/generateMagiskLink.py @@ -25,21 +25,18 @@ import json import requests from pathlib import Path -magisk_apk = sys.argv[1] +magisk_ver = sys.argv[1] download_dir = Path.cwd().parent / "download" if sys.argv[2] == "" else Path(sys.argv[2]).resolve() tempScript = sys.argv[3] -print(f"Generating Magisk download link: release type={magisk_apk}", flush=True) -if not magisk_apk: - magisk_apk = "stable" -if magisk_apk == "stable" or magisk_apk == "beta" or magisk_apk == "canary" or magisk_apk == "debug": - magisk_apk = json.loads(requests.get( - f"https://github.com/topjohnwu/magisk-files/raw/master/{magisk_apk}.json").content)['magisk']['link'] -print(f"download link: {magisk_apk}", flush=True) -out_file = download_dir / "magisk.zip" +print(f"Generating Magisk download link: release type={magisk_ver}", flush=True) +if not magisk_ver: + magisk_ver = "stable" +if magisk_ver == "stable" or magisk_ver == "beta" or magisk_ver == "canary" or magisk_ver == "debug": + magisk_link = json.loads(requests.get( + f"https://github.com/topjohnwu/magisk-files/raw/master/{magisk_ver}.json").content)['magisk']['link'] +print(f"download link: {magisk_link}", flush=True) -if not os.path.isfile(out_file): - # urllib.request.urlretrieve(magisk_apk, out_file) - with open(download_dir/tempScript, 'a') as f: - f.writelines(f'{magisk_apk}\n') - f.writelines(f' dir={download_dir}\n') - f.writelines(f' out=magisk.zip\n') +with open(download_dir/tempScript, 'a') as f: + f.writelines(f'{magisk_link}\n') + f.writelines(f' dir={download_dir}\n') + f.writelines(f' out=magisk-{magisk_ver}.zip\n') diff --git a/scripts/generateWSALinks.py b/scripts/generateWSALinks.py index 8d6dd17..4d761c4 100644 --- a/scripts/generateWSALinks.py +++ b/scripts/generateWSALinks.py @@ -36,6 +36,7 @@ release_type_map = {"retail": "Retail", "release preview": "RP", "insider slow": "WIS", "insider fast": "WIF"} release_type = release_type_map[sys.argv[2]] if sys.argv[2] != "" else "Retail" download_dir = Path.cwd().parent / "download" if sys.argv[3] == "" else Path(sys.argv[3]).resolve() +download_dir tempScript = sys.argv[4] cat_id = '858014f3-3934-4abe-8078-4aa193e74ca8' print(f"Generating WSA download link: arch={arch} release_type={release_type}", flush=True) @@ -85,14 +86,14 @@ if not os.path.exists(download_dir): tmpdownlist = open(download_dir/tempScript, 'a') for i, v, f in identities: if re.match(f"Microsoft\.UI\.Xaml\..*_{arch}_.*\.appx", f): - out_file = download_dir / "xaml.appx" - out_file_name = "xaml.appx" + out_file_name = f"xaml-{arch}.appx" + out_file = download_dir / out_file_name # elif re.match(f"Microsoft\.VCLibs\..+\.UWPDesktop_.*_{arch}_.*\.appx", f): # out_file = download_dir / "vclibs.appx" # out_file_name = "vclibs.appx" elif re.match(f"MicrosoftCorporationII\.WindowsSubsystemForAndroid_.*\.msixbundle", f): - out_file = download_dir / "wsa.zip" - out_file_name = "wsa.zip" + out_file_name = f"wsa-{arch}-{release_type}.zip" + out_file = download_dir / out_file_name else: continue out = requests.post( @@ -105,13 +106,11 @@ for i, v, f in identities: for l in doc.getElementsByTagName("FileLocation"): url = l.getElementsByTagName("Url")[0].firstChild.nodeValue if len(url) != 99: - if not os.path.isfile(out_file): - print(f"download link: {url} to {out_file}", flush=True) - # urllib.request.urlretrieve(url, out_file) - tmpdownlist.writelines(url + '\n') - tmpdownlist.writelines(f' dir={download_dir}\n') - tmpdownlist.writelines(f' out={out_file_name}\n') + print(f"download link: {url} to {out_file}", flush=True) + tmpdownlist.writelines(url + '\n') + tmpdownlist.writelines(f' dir={download_dir}\n') + tmpdownlist.writelines(f' out={out_file_name}\n') tmpdownlist.writelines(f'https://aka.ms/Microsoft.VCLibs.{arch}.14.00.Desktop.appx\n') tmpdownlist.writelines(f' dir={download_dir}\n') -tmpdownlist.writelines(f' out=vclibs.appx\n') +tmpdownlist.writelines(f' out=vclibs-{arch}.appx\n') tmpdownlist.close diff --git a/scripts/run.sh b/scripts/run.sh index 9a4b897..ae94eae 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -34,7 +34,6 @@ DOWNLOAD_DIR=../download DOWNLOAD_CONF_NAME=download.list OUTPUT_DIR=../output MOUNT_DIR="$WORK_DIR"/system -CLEAN_DOWNLOAD=1 umount_clean(){ echo "Cleanup Work Directory" if [ -d "$MOUNT_DIR" ]; then @@ -51,19 +50,26 @@ umount_clean(){ fi sudo rm -rf "${WORK_DIR:?}" } +clean_download(){ + if [ -d "$DOWNLOAD_DIR" ]; then + echo "Cleanup Download Directory" + if [ "$CLEAN_DOWNLOAD_WSA" = "1" ]; then + rm -f "${WSA_ZIP_PATH:?}" + fi + if [ "$CLEAN_DOWNLOAD_MAGISK" = "1" ]; then + rm -f "${MAGISK_PATH:?}" + fi + if [ "$CLEAN_DOWNLOAD_GAPPS" = "1" ]; then + rm -f "${GAPPS_PATH:?}" + fi + fi +} abort() { echo "An error has occurred, exit" if [ -d "$WORK_DIR" ]; then umount_clean fi - if [ -d "$DOWNLOAD_DIR" ] && [ $CLEAN_DOWNLOAD = "1" ]; then - echo "Cleanup Download Directory" - sudo rm -rf "${DOWNLOAD_DIR:?}" - fi - if [ -d "$OUTPUT_DIR" ]; then - echo "Cleanup Output Directory" - sudo rm -rf "${OUTPUT_DIR:?}" - fi + clean_download exit 1 } trap abort INT TERM @@ -121,7 +127,7 @@ MAGISK_VER=$( ) if (YesNoBox '([title]="Install GApps" [text]="Do you want to install GApps?")'); then - if [ -f "$DOWNLOAD_DIR"/MindTheGapps/MindTheGapps_"$ARCH".zip ]; then + if [ -f "$DOWNLOAD_DIR"/MindTheGapps-"$ARCH".zip ]; then GAPPS_BRAND=$( Radiolist '([title]="Which GApps do you want to install?" [default]="OpenGApps")' \ @@ -176,7 +182,8 @@ fi clear echo -e "ARCH=$ARCH\nRELEASE_TYPE=$RELEASE_TYPE\nMAGISK_VER=$MAGISK_VER\nGAPPS_VARIANT=$GAPPS_VARIANT\nREMOVE_AMAZON=$REMOVE_AMAZON\nROOT_SOL=$ROOT_SOL\nCOMPRESS_OUTPUT=$COMPRESS_OUTPUT" - +declare -A RELEASE_TYPE_MAP=(["retail"]="Retail" ["release preview"]="RP" ["insider slow"]="WIS" ["insider fast"]="WIF") +trap 'rm -f -- "${DOWNLOAD_DIR:?}/${DOWNLOAD_CONF_NAME}"' EXIT echo "Generate Download Links" python3 generateWSALinks.py "$ARCH" "$RELEASE_TYPE" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort python3 generateMagiskLink.py "$MAGISK_VER" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort @@ -187,35 +194,65 @@ if [ $GAPPS_VARIANT != 'none' ] && [ $GAPPS_VARIANT != '' ]; then # python3 generateGappsLink.py "$ARCH" "$GAPPS_VARIANT" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort fi fi -trap 'rm -f -- "${DOWNLOAD_DIR:?}/${DOWNLOAD_CONF_NAME}"' EXIT echo "Download Artifacts" -if ! aria2c --no-conf --log-level=info --log="$DOWNLOAD_DIR/aria2_download.log" -x16 -s16 -j5 -c -R -m0 -d"$DOWNLOAD_DIR" -i"$DOWNLOAD_DIR"/"$DOWNLOAD_CONF_NAME"; then +if ! aria2c --no-conf --log-level=info --log="$DOWNLOAD_DIR/aria2_download.log" -x16 -s16 -j5 -c -R -m0 --allow-overwrite=true --conditional-get=true -d"$DOWNLOAD_DIR" -i"$DOWNLOAD_DIR"/"$DOWNLOAD_CONF_NAME"; then echo "We have encountered an error while downloading files." exit 1 -else - CLEAN_DOWNLOAD=0 fi +WSA_ZIP_PATH=$DOWNLOAD_DIR/wsa-$ARCH-${RELEASE_TYPE_MAP[$RELEASE_TYPE]}.zip +MAGISK_PATH=$DOWNLOAD_DIR/magisk-$MAGISK_VER.zip +GAPPS_PATH="$DOWNLOAD_DIR"/OpenGApps-$ARCH-$GAPPS_VARIANT.zip echo "Extract WSA" -WSA_WORK_ENV="${WORK_DIR:?}"/ENV -if [ -f "$WSA_WORK_ENV" ]; then rm -f "${WSA_WORK_ENV:?}"; fi -export WSA_WORK_ENV -python3 extractWSA.py "$ARCH" "$WORK_DIR" || abort -echo -e "Extract done\n" -source "${WORK_DIR:?}/ENV" || abort - +if [ -f "$WSA_ZIP_PATH" ]; then + WSA_WORK_ENV="${WORK_DIR:?}"/ENV + if [ -f "$WSA_WORK_ENV" ]; then rm -f "${WSA_WORK_ENV:?}"; fi + export WSA_WORK_ENV + if ! python3 extractWSA.py "$ARCH" "$WSA_ZIP_PATH" "$WORK_DIR"; then + echo "Unzip WSA failed, is the download incomplete?" + CLEAN_DOWNLOAD_WSA=1 + abort + fi + echo -e "Extract done\n" + source "${WORK_DIR:?}/ENV" || abort +else + echo "The WSA zip package does not exist, is the download incomplete?" + exit 1 +fi echo "Extract Magisk" -python3 extractMagisk.py "$ARCH" "$DOWNLOAD_DIR/magisk.zip" "$WORK_DIR" || abort + +if [ -f "$MAGISK_PATH" ]; then + if ! python3 extractMagisk.py "$ARCH" "$MAGISK_PATH" "$WORK_DIR"; then + echo "Unzip Magisk failed, is the download incomplete?" + CLEAN_DOWNLOAD_MAGISK=1 + abort + fi +else + echo "The Magisk zip package does not exist, is the download incomplete?" + exit 1 +fi echo -e "done\n" if [ $GAPPS_VARIANT != 'none' ] && [ $GAPPS_VARIANT != '' ]; then echo "Extract GApps" mkdir -p "$WORK_DIR"/gapps || abort if [ $GAPPS_BRAND = "OpenGApps" ]; then - unzip -p "$DOWNLOAD_DIR"/gapps.zip {Core,GApps}/'*.lz' | tar --lzip -C "$WORK_DIR"/gapps -xf - -i --strip-components=2 --exclude='setupwizardtablet-x86_64' --exclude='packageinstallergoogle-all' --exclude='speech-common' --exclude='markup-lib-arm' --exclude='markup-lib-arm64' --exclude='markup-all' --exclude='setupwizarddefault-x86_64' --exclude='pixellauncher-all' --exclude='pixellauncher-common' || abort + if [ -f "$GAPPS_PATH" ]; then + if ! unzip -p "$GAPPS_PATH" {Core,GApps}/'*.lz' | tar --lzip -C "$WORK_DIR"/gapps -xf - -i --strip-components=2 --exclude='setupwizardtablet-x86_64' --exclude='packageinstallergoogle-all' --exclude='speech-common' --exclude='markup-lib-arm' --exclude='markup-lib-arm64' --exclude='markup-all' --exclude='setupwizarddefault-x86_64' --exclude='pixellauncher-all' --exclude='pixellauncher-common'; then + echo "Unzip GApps failed, is the download incomplete?" + CLEAN_DOWNLOAD_GAPPS=1 + abort + fi + else + echo "The WSA zip package does not exist, is the download incomplete?" + exit 1 + fi else - unzip "$DOWNLOAD_DIR"/MindTheGapps/MindTheGapps_"$ARCH".zip "system/*" -x "system/addon.d/*" "system/system_ext/priv-app/SetupWizard/*" -d "$WORK_DIR"/gapps || abort + if ! unzip "$DOWNLOAD_DIR"/MindTheGapps-"$ARCH".zip "system/*" -x "system/addon.d/*" "system/system_ext/priv-app/SetupWizard/*" -d "$WORK_DIR"/gapps; then + echo "Unzip MindTheGapps failed, package is corrupted?" + abort + fi mv "$WORK_DIR"/gapps/system/* "$WORK_DIR"/gapps || abort rm -rf "${WORK_DIR:?}"/gapps/system || abort fi @@ -444,6 +481,7 @@ if [ $GAPPS_VARIANT != 'none' ] && [ $GAPPS_VARIANT != '' ]; then find "$WORK_DIR"/gapps/product/framework/ -maxdepth 1 -mindepth 1 -printf '%P\n' | xargs -I item sudo find "$MOUNT_DIR"/product/framework/item -type f -exec chcon --reference="$MOUNT_DIR"/product/etc/permissions/com.android.settings.intelligence.xml {} \; find "$WORK_DIR"/gapps/system_ext/etc/permissions/ -maxdepth 1 -mindepth 1 -printf '%P\n' | xargs -I file sudo find "$MOUNT_DIR"/system_ext/etc/permissions/file -type f -exec chcon --reference="$MOUNT_DIR"/system_ext/etc/permissions/com.android.systemui.xml {} \; + sudo chcon --reference="$MOUNT_DIR"/product/lib64/libjni_eglfence.so "$MOUNT_DIR"/product/lib find "$WORK_DIR"/gapps/product/lib/ -maxdepth 1 -mindepth 1 -printf '%P\n' | xargs -I file sudo find "$MOUNT_DIR"/product/lib/file -exec chcon --reference="$MOUNT_DIR"/product/lib64/libjni_eglfence.so {} \; find "$WORK_DIR"/gapps/product/lib64/ -maxdepth 1 -mindepth 1 -printf '%P\n' | xargs -I file sudo find "$MOUNT_DIR"/product/lib64/file -type f -exec chcon --reference="$MOUNT_DIR"/product/lib64/libjni_eglfence.so {} \; find "$WORK_DIR"/gapps/system_ext/priv-app/ -maxdepth 1 -mindepth 1 -printf '%P\n' | xargs -I dir sudo find "$MOUNT_DIR"/system_ext/priv-app/dir -type d -exec chcon --reference="$MOUNT_DIR"/system_ext/priv-app {} \; @@ -487,7 +525,7 @@ echo -e "Shrink images done\n" echo "Remove signature and add scripts" sudo rm -rf "${WORK_DIR:?}"/wsa/"$ARCH"/\[Content_Types\].xml "$WORK_DIR"/wsa/"$ARCH"/AppxBlockMap.xml "$WORK_DIR"/wsa/"$ARCH"/AppxSignature.p7x "$WORK_DIR"/wsa/"$ARCH"/AppxMetadata || abort -cp "$DOWNLOAD_DIR"/vclibs.appx "$DOWNLOAD_DIR"/xaml.appx "$WORK_DIR"/wsa/"$ARCH" || abort +cp "$DOWNLOAD_DIR"/vclibs-"$ARCH".appx "$DOWNLOAD_DIR"/xaml-"$ARCH".appx "$WORK_DIR"/wsa/"$ARCH" || abort tee "$WORK_DIR"/wsa/"$ARCH"/Install.ps1 <