Skip to content

Update shell scripts #686

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wiryhatbox wants to merge 6 commits into
base: master
Choose a base branch
from
+159 −122
Open

Update shell scripts #686

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
17ce47c
Convert to sh for portability
wiryhatbox Jul 4, 2024
cf6420a
Use `set -euf` to make script safer
wiryhatbox Jul 4, 2024
c853972
Use `set -euf` to make script safer
wiryhatbox Jul 4, 2024
c6537f7
Convert to sh for portability
wiryhatbox Jul 5, 2024
0035011
Use `set -euf` to make script safer
wiryhatbox Jul 5, 2024
0aac229
Use `set -euf` to make script safer
wiryhatbox Jul 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 48 additions & 35 deletions checkBuild.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
#!/bin/bash
#!/bin/sh

if [[ $# -eq 0 ]] ; then
set -euf

die () { echo "Error: $*"; exit 1; }

if [ $# -eq 0 ]; then
printf "No arguments supplied.\nUsage: ./checkBuild.sh FILE REVISION [RETRIES]\n\n \
FILE The apk you want to verify, without extension. mbw-prodnet-release for example\n \
REVISION The git revision, tag or branch. v2.12.0.19 for example.\n \
Expand All @@ -16,90 +20,99 @@ revision=$2
# with every iteration. Pick how many retries you want to run:
retries=${3:-0}

if [[ ! -f "./${releaseFile}.apk" ]]
if [ ! -f "./${releaseFile}.apk" ]
then
echo "./${releaseFile}.apk not found. Put the file you want to test into this folder and reference it omitting the \".apk\"."
exit 1
die "./${releaseFile}.apk not found. Put the file you want to test into this folder and reference it omitting the \".apk\"."
fi

if [[ ! -f "apktool.jar" ]]
if [ ! -f "apktool.jar" ]
then
echo "apktool.jar not found. Put the file into this folder. You can get it from https://ibotpeaches.github.io/Apktool/ or https://github.com/iBotPeaches/Apktool"
exit 1
die "apktool.jar not found. Put the file into this folder. You can get it from https://ibotpeaches.github.io/Apktool/ or https://github.com/iBotPeaches/Apktool"
fi

workdir=/tmp/mbwDeterministicBuild/
mkdir -p $workdir
mkdir -p -- "$workdir" || die "Cannot create directory $workdir"

printf "Checking $releaseFile against revision $revision with $retries retries if verification fails.\n \
You can monitor progress in ${workdir}progress.log\n" | tee ${workdir}progress.log
printf "%s\n%s\n" "Checking $releaseFile against revision $revision with $retries retries if verification fails." \
"You can monitor progress in ${workdir}progress.log" | tee "${workdir}progress.log"

cp -r . $workdir
cd $workdir
if ! cp -r -- . "$workdir" || ! cd -- "$workdir"
then
die "Error: Cannot cp and cd to $workdir (maybe no disk space?)"
fi

rm local.properties # this doesn't work in docker and shouldn't be needed outside of docker.
rm -f -- local.properties || true # this doesn't work in docker and shouldn't be needed outside of docker.

java -jar apktool.jar d --output original $releaseFile.apk
java -jar apktool.jar d --output original "$releaseFile.apk"

git config user.email "[email protected]"
git config user.name "only temporary"
git stash
git checkout $revision
git checkout -- "$revision" || die "Invalid commit."
sed -i 's/[email protected]:/https:\/\/github.com\//' .gitmodules
git submodule update --init --recursive
git submodule update --init --recursive || die "Cannot initialize submodules."

# these files are either irrelevant (apktool.yml is created by the akp extractor) or not reproducible signature/meta data (CERT, MANIFEST)
ignoreFiles="apktool.yml\|original/META-INF/CERT.RSA\|original/META-INF/CERT.SF\|original/META-INF/MANIFEST.MF"

./gradlew clean :mbw:assProdRel
cp ./mbw/build/outputs/apk/prodnet/release/mbw-prodnet-release.apk candidate.apk
cp ./mbw/build/outputs/apk/prodnet/release/mbw-prodnet-release.apk candidate.apk ||
die "Cannot copy a file (maybe no disk space?)"

if [ ! -e candidate.apk ]
then
echo "Unexpected error: candidate.apk doesn't exist. Possibly the build failed. Check and try again."
exit 1
die "Unexpected error: candidate.apk doesn't exist. Possibly the build failed. Check and try again."
fi

java -jar apktool.jar d candidate.apk
set +x
diff --brief --recursive original/ candidate/ > 0.diff
set -x
mv candidate/ 0/
# store the list of relevant differing files in remaining.diff
cat 0.diff | grep -v "$ignoreFiles" > remaining.diff
set +x
grep -v -- "$ignoreFiles" 0.diff > remaining.diff
set -x

checkFinished() {
# if remaining.diff is empty, we are done.
if [ ! -s remaining.diff ]
then
printf "$releaseFile matches $revision!\n" >> progress.log
printf "%s\n" "$releaseFile matches $revision!" >> progress.log
cat progress.log
exit 0
else
printf "$(cat remaining.diff | wc -l) files differ. Trying harder to find matches.\n \
Remaining files:\n$(cat remaining.diff)\n\n" >> progress.log
printf "%s\n%s\n%s\n\n" "$(wc -l remaining.diff) files differ. Trying harder to find matches." \
"Remaining files:" "$(cat remaining.diff)" >> progress.log
fi
}

checkFinished

for i in $( seq 1 $retries ) ; do
printf "Deterministic build failed. Attempt $i to still confirm all files individually under not totally deterministic conditions.\n" >> progress.log
for i in $( seq 1 "$retries" ) ; do
printf "%s\n" "Deterministic build failed. Attempt $i to still confirm all files individually under not totally deterministic conditions." >> progress.log
./gradlew clean :mbw:assProdRel
cp ./mbw/build/outputs/apk/prodnet/release/mbw-prodnet-release.apk candidate.apk
cp ./mbw/build/outputs/apk/prodnet/release/mbw-prodnet-release.apk candidate.apk ||
die "Cannot copy a file (maybe no disk space?)"
java -jar apktool.jar d candidate.apk
# join remaining filenames to a grep filter. Only these file names are relevant.
relevantFilesFilter=$(cat remaining.diff | paste -sd "|" - | sed 's/|/\\\|/g')
relevantFilesFilter="$(paste -sd "|" remaining.diff | sed 's/|/\\\|/g')"
# diff, only listing files that differ
diff --brief --recursive original/ candidate/ > $i.diff
mv candidate/ $i/
set +x
diff --brief --recursive original/ candidate/ > "$i.diff"
set -x
mv -- candidate/ "$i/"
# diff lines that were present in all prior diffs get stored in remaining.diff
cat $i.diff | grep "$relevantFilesFilter" > remaining.diff
set +x
grep -- "$relevantFilesFilter" "$i.diff" > remaining.diff
set -x
checkFinished
done

printf "Error: The following files could not be verified:\n \
$( cat remaining.diff ) \
Error: Giving up after $retries retries. Please manually check if the differing files are acceptable using meld or any other folder diff tool.\n \
Not every tiny diff is a red flag. Maybe this script is broken. Maybe a tool compiles in a non-deterministic way. Both happened before." \
printf "%s\n%s%s\n%s" "Error: The following files could not be verified:" \
"$(cat remaining.diff)" \
"Error: Giving up after $retries retries. Please manually check if the differing files are acceptable using meld or any other folder diff tool." \
"Not every tiny diff is a red flag. Maybe this script is broken. Maybe a tool compiles in a non-deterministic way. Both happened before." \
| tee -a progress.log

exit 1
25 changes: 16 additions & 9 deletions collectApks.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,19 @@

# this script helps wrap up the currently 8 apks into 2 zips, named containing the versionName

releasefolder="/tmp/release_mbw/comp_"$(date +"%s")"/"
mbwVersion=$( grep "versionName '" mbw/build.gradle | sed "s/.*versionName //g" | sed "s/'//g" )
mkdir -p $releasefolder
for f in $( find . -name *.apk ); do
echo $f
cp $f $releasefolder
done
zip $releasefolder"release_mbw_${mbwVersion}.zip" ${releasefolder}mbw*.apk >/dev/null 2>&1
echo done
set -euf

releasefolder="/tmp/release_mbw/comp_$(date +%s)/"
mbwVersion="$( grep "versionName '" mbw/build.gradle | sed "s/.*versionName //g" | sed "s/'//g" )"
mkdir -p -- "$releasefolder"
if ! find . -name "*.apk" -exec echo {} \; -exec cp -- {} "$releasefolder" \; ; then
printf "Error: Cannot copy a file (maybe no disk space?)"
exit 1
fi

if ! zip "${releasefolder}release_mbw_${mbwVersion}.zip" "${releasefolder}mbw*.apk" >/dev/null 2>&1; then
printf "Error: Cannot zip a file (maybe no disk space?)"
exit 1
fi

printf "Done\n"
76 changes: 38 additions & 38 deletions gradlew
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 15 additions & 18 deletions mbw/res-sources/updateDrawables.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,36 +1,33 @@
#!/bin/bash

set -euf -o pipefail

die () { echo "Error: $*"; exit 1; }

sizeNames=(ldpi mdpi hdpi xhdpi xxhdpi)
sizes=(360 480 720 960 1800)
fileIn=localTraderLocalOnly.png
fileOut=lt_local_only_warning.png

for ((n=0; n <= 4; n++))
do
fOut="../src/main/res/drawable-"${sizeNames[n]}"/"$fileOut
convert -background none -resize ${sizes[n]}"x" $fileIn - | \
pngquant --force 64 > $fOut
done
conv_function () {
for ((n=0; n <= 4; n++))
do
fOut="../src/main/res/drawable-${sizeNames[n]}/${fileOut}"
convert -background none -resize "${sizes[n]}x" "$fileIn" - | \
pngquant --force 64 > "$fOut" || die "Command failed."
done
}

conv_function || die "Command failed."

sizes=(100 133 200 267 400)
fileIn=creditCard.png
fileOut=credit_card_buy.png

for ((n=0; n <= 4; n++))
do
fOut="../src/main/res/drawable-"${sizeNames[n]}"/"$fileOut
convert -background none -resize ${sizes[n]}"x" $fileIn - | \
pngquant --force 64 > $fOut
done
conv_function || die "Command failed."

sizes=(100 133 200 267 400)
fileIn=mycelium_logo_transp.png
fileOut=mycelium_logo_transp.png

for ((n=0; n <= 4; n++))
do
fOut="../src/main/res/drawable-"${sizeNames[n]}"/"$fileOut
convert -background none -resize ${sizes[n]}"x" $fileIn - | \
pngquant --force 64 > $fOut
done
conv_function || die "Command failed."
Loading