Mac M1 CI/CD (#111)

* fix shellcheck warnings

* fix mistake

* add static .DS_Store and other DMG styling files

* fix architecture

* fix build_mac.sh

* Unifying qflipper_common.pri and build_mac.sh, add fixRpath.py

* Add static build(temp)

* Add static build for M1

* add M1 workflow

* fix ci.yml

* add missed file application.pro

* Add dual-architecture build

* Static qt (#115)

* Clean up qmake files

* Add conditions for static build

* Add versionMajor() plugin method

* More conditional compilation statements for static build

* fix style in build_mac.sh, suppressing LD warnings

* Remove separate CI for Mac

* add keychain unlock inside CI

* fix slash in branch name

* Fix path in update server

* Fix path in update server again

* fix rsync flags

* upgrade rsync action

* fix ci.yml

* fix path on update server

* relink x86 version of libusb too

* Replace create-dmg to submodule

* Add CONFIG var

* Add new libusb, and it's build script

Co-authored-by: Georgii Surkov <37121527+gsurkov@users.noreply.github.com>
This commit is contained in:
Max Andreev 2022-09-09 12:25:01 +03:00 committed by GitHub
parent 2a773ecd4a
commit 124251bf2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 230 additions and 121 deletions

View File

@ -36,8 +36,10 @@ jobs:
retention-days: 1
build_mac:
runs-on: [self-hosted, X64, macOS]
runs-on: [self-hosted, ARM64, macOS]
env:
MAC_OS_KEYCHAIN_NAME: ${{ secrets.MAC_OS_KEYCHAIN_NAME }}
MAC_OS_KEYCHAIN_PASSWORD: ${{ secrets.MAC_OS_KEYCHAIN_PASSWORD }}
MAC_OS_SIGNING_KEY_ID: ${{ secrets.MAC_OS_SIGNING_KEY_ID }}
MAC_OS_SIGNING_BUNDLE_ID: ${{ secrets.MAC_OS_SIGNING_BUNDLE_ID }}
MAC_OS_SIGNING_ASC_PROVIDER: ${{ secrets.MAC_OS_SIGNING_ASC_PROVIDER }}
@ -88,7 +90,7 @@ jobs:
upload:
name: Upload apps to storage
needs: [build_windows, build_mac,build_linux]
needs: [build_windows, build_mac, build_linux]
runs-on: ubuntu-latest
steps:
@ -104,11 +106,15 @@ jobs:
- name: 'Generate tag suffix'
if: startsWith(github.ref, 'refs/tags/') == true
run: echo "SUFFIX=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
run: |
echo "REF=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
echo "SUFFIX=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
- name: 'Generate branch suffix'
if: startsWith(github.ref, 'refs/tags/') != true
run: echo "SUFFIX=$(git rev-parse --abbrev-ref HEAD)-$(date +'%d%m%Y')-$(git rev-parse --short HEAD)" >> $GITHUB_ENV
run: |
echo "REF=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV
echo "SUFFIX=$(git rev-parse --abbrev-ref HEAD | sed 's/\//_/g')-$(date +'%d%m%Y')-$(git rev-parse --short HEAD)" >> $GITHUB_ENV
- uses: actions/download-artifact@v2
with:
@ -121,19 +127,18 @@ jobs:
name: qflipper.dmg
- name: Rename app
run:
|
run: |
mv qFlipper-x86_64.AppImage ./artifacts/qFlipper-x86_64-${SUFFIX}.AppImage
mv qFlipperSetup-64bit.exe ./artifacts/qFlipperSetup-64bit-${SUFFIX}.exe
mv qFlipper-64bit.zip ./artifacts/qFlipper-64bit-${SUFFIX}.zip
mv qflipper.dmg ./artifacts/qFlipper-${SUFFIX}.dmg
- name: Upload apps
uses: burnett01/rsync-deployments@4.1
uses: burnett01/rsync-deployments@5.1
with:
switches: -cvzr --delete
switches: -cvzr --delete --mkpath
path: artifacts/
remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}qFlipper/${GITHUB_REF##*/}/"
remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}qFlipper/${REF}/"
remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}

1
.gitignore vendored
View File

@ -5,3 +5,4 @@ build
x64
Win32
.vs
.DS_Store

3
.gitmodules vendored
View File

@ -4,3 +4,6 @@
[submodule "3rdparty/nanopb"]
path = 3rdparty/nanopb
url = https://github.com/nanopb/nanopb.git
[submodule "scripts/create-dmg"]
path = scripts/create-dmg
url = https://github.com/flipperdevices/create-dmg

View File

@ -29,27 +29,16 @@ TRANSLATIONS += \
CONFIG += lrelease
CONFIG += embed_translations
# Additional import path used to resolve QML modules in Qt Creator's code model
QML_IMPORT_PATH =
# Additional import path used to resolve QML modules just for Qt Quick Designer
QML_IMPORT_PATH = $$PWD/imports
QML_IMPORT_PATH += $$PWD/imports
unix|win32 {
LIBS += \
-L$$OUT_PWD/../backend/ -lbackend \
-L$$OUT_PWD/../dfu/ -ldfu
}
win32:!win32-g++ {
PRE_TARGETDEPS += \
$$OUT_PWD/../backend/backend.lib \
$$OUT_PWD/../dfu/dfu.lib
} else:unix|win32-g++ {
PRE_TARGETDEPS += \
$$OUT_PWD/../backend/libbackend.a \
$$OUT_PWD/../dfu/libdfu.a
contains(CONFIG, static): LIBS += \
-L$$OUT_PWD/../3rdparty/ -l3rdparty \
-L$$OUT_PWD/../plugins/ -lflipperproto0
}
win32 {
@ -74,10 +63,6 @@ INCLUDEPATH += \
$$PWD/../dfu \
$$PWD/../backend
DEPENDPATH += \
$$PWD/../dfu \
$$PWD/../backend
HEADERS += \
application.h \
applicationupdater.h \

View File

@ -209,18 +209,5 @@ HEADERS += \
updateregistry.h \
versioninfo.h
unix|win32 {
LIBS += -L$$OUT_PWD/../dfu/ -ldfu
}
win32:!win32-g++ {
PRE_TARGETDEPS += $$OUT_PWD/../dfu/dfu.lib
} else:unix|win32-g++ {
PRE_TARGETDEPS += $$OUT_PWD/../dfu/libdfu.a
}
INCLUDEPATH += $$PWD/../dfu \
$$PWD/../plugins/protobufinterface
DEPENDPATH += $$PWD/../dfu

View File

@ -36,6 +36,10 @@
#include "rpc/guistartvirtualdisplayoperation.h"
#include "rpc/guistopvirtualdisplayoperation.h"
#if defined(QT_STATIC)
Q_IMPORT_PLUGIN(ProtobufPlugin)
#endif
Q_LOGGING_CATEGORY(LOG_SESSION, "RPC")
using namespace Flipper;
@ -46,7 +50,9 @@ ProtobufSession::ProtobufSession(const QSerialPortInfo &portInfo, QObject *paren
m_sessionState(Stopped),
m_portInfo(portInfo),
m_serialPort(nullptr),
#if !defined(QT_STATIC)
m_loader(new QPluginLoader(this)),
#endif
m_plugin(nullptr),
m_currentOperation(nullptr),
m_bytesToWrite(0),
@ -259,10 +265,12 @@ void ProtobufSession::onSerialPortReadyRead()
if(!isSessionUp()) {
m_serialPort->clear();
return;
#if !defined(QT_STATIC)
} else if(!m_loader->isLoaded()) {
// For some weird reason the plugin can be unloaded by connecting
// multiple devices (although the docs say it shouldn't)
loadProtobufPlugin();
#endif
}
m_receivedData.append(m_serialPort->readAll());
@ -329,10 +337,12 @@ void ProtobufSession::writeToPort()
{
if(!m_currentOperation || !m_plugin) {
return;
#if !defined(QT_STATIC)
} else if(!m_loader->isLoaded()) {
// For some weird reason the plugin can be unloaded by connecting
// multiple devices (although the docs say it shouldn't)
loadProtobufPlugin();
#endif
}
bool success;
@ -409,6 +419,7 @@ void ProtobufSession::setSessionState(SessionState newState)
emit sessionStateChanged();
}
#if !defined(QT_STATIC)
const QString ProtobufSession::protobufPluginFileName(int versionMajor)
{
#if defined(Q_OS_WINDOWS)
@ -421,10 +432,23 @@ const QString ProtobufSession::protobufPluginFileName(int versionMajor)
#error "Unsupported OS"
#endif
}
#endif
QVector<int> ProtobufSession::supportedProtobufVersions()
{
QVector<int> ret;
#if defined(QT_STATIC)
const auto staticInstances = QPluginLoader::staticInstances();
for(const auto *instance: staticInstances) {
if(const auto *protobufPlugin = qobject_cast<const ProtobufPluginInterface*>(instance)) {
ret.append(protobufPlugin->versionMajor());
}
}
std::sort(ret.begin(), ret.end());
#else
const auto libraryPaths = QCoreApplication::libraryPaths();
for(auto i = 0;; ++i) {
@ -441,7 +465,7 @@ QVector<int> ProtobufSession::supportedProtobufVersions()
break;
}
}
#endif
return ret;
}
@ -458,21 +482,36 @@ bool ProtobufSession::loadProtobufPlugin()
return false;
}
m_loader->setFileName(protobufPluginFileName(m_versionMajor));
#if defined(QT_STATIC)
const auto staticInstances = QPluginLoader::staticInstances();
if(!(m_plugin = qobject_cast<ProtobufPluginInterface*>(m_loader->instance()))) {
qCCritical(LOG_SESSION) << "Failed to load protobuf plugin:" << m_loader->errorString();
} else {
m_plugin->setMinorVersion(m_versionMinor);
for(auto *instance : staticInstances) {
if(auto *protobufPlugin = qobject_cast<ProtobufPluginInterface*>(instance)) {
if(protobufPlugin->versionMajor() == m_versionMajor) {
m_plugin = protobufPlugin;
break;
}
}
}
#else
m_loader->setFileName(protobufPluginFileName(m_versionMajor));
m_plugin = qobject_cast<ProtobufPluginInterface*>(m_loader->instance());
return m_plugin;
if(!m_plugin) {
qCCritical(LOG_SESSION) << "Failed to load protobuf plugin:" << m_loader->errorString();
return false;
}
#endif
m_plugin->setMinorVersion(m_versionMinor);
return true;
}
void ProtobufSession::unloadProtobufPlugin()
{
m_plugin = nullptr;
#if defined(QT_SHARED)
if(!m_loader->isLoaded()) {
return;
}
@ -484,6 +523,7 @@ void ProtobufSession::unloadProtobufPlugin()
} else {
qCDebug(LOG_SESSION) << "Unloaded protobuf plugin.";
}
#endif
}
void ProtobufSession::stopEarly(BackendError::ErrorType error, const QString &errorString)

View File

@ -108,7 +108,9 @@ private slots:
void onCurrentOperationFinished();
private:
#if !defined(QT_STATIC)
static const QString protobufPluginFileName(int versionMajor);
#endif
static QVector<int> supportedProtobufVersions();
void setSessionState(SessionState newState);
@ -136,7 +138,9 @@ private:
QSerialPort *m_serialPort;
QByteArray m_receivedData;
#if !defined(QT_STATIC)
QPluginLoader *m_loader;
#endif
ProtobufPluginInterface *m_plugin;
QQueue<AbstractProtobufOperation*> m_queue;
AbstractProtobufOperation *m_currentOperation;

View File

@ -1,67 +1,102 @@
#!/bin/bash
set -e
set -x
# shellcheck disable=SC2207
PROJECT_DIR=`pwd`
PROJECT="qFlipper"
BUILD_DIRECTORY="build_mac"
set -exuo pipefail;
if [[ -d "$BUILD_DIRECTORY" ]]
then
rm -rf "$BUILD_DIRECTORY"
PROJECT="qFlipper";
BUILD_DIRECTORY="build_mac";
LIBUSB_PATH="/opt/libs/libusb/1.0.24-universal";
if [ -d ".git" ]; then
git submodule update --init;
fi
mkdir "$BUILD_DIRECTORY"
cd "$BUILD_DIRECTORY"
qmake -spec macx-clang CONFIG+=release CONFIG+=x86_64 -o Makefile ../$PROJECT.pro
make qmake_all && make -j9 > /dev/null && make install
macdeployqt $PROJECT.app -executable=$PROJECT.app/Contents/MacOS/${PROJECT}-cli -qmldir=$PROJECT_DIR/Application -verbose=1
FAILED_LIBS_COUNT=`otool -L $PROJECT.app/Contents/Frameworks/*.dylib | grep /usr/local -c || true`
FAILED_APPS_COUNT=`otool -L $PROJECT.app/Contents/MacOS/* | grep /usr/local -c || true`
if [[ $FAILED_LIBS_COUNT -gt 0 ]]
then
echo "Not all libraries use proper paths"
exit 255
elif [[ $FAILED_APPS_COUNT -gt 0 ]]
then
echo "Not all executables use proper paths"
exit 255
if [[ "$(uname -s)" != "Darwin" ]]; then
echo "This script needs to be runned under MacOS";
exit 1;
fi
if [[ "$(uname -m)" == "arm64" ]]; then
eval "$(/opt/homebrew/bin/brew shellenv)";
PATH="/opt/homebrew/qt-6.3.1-static/bin:$PATH";
export PKG_CONFIG_PATH="$LIBUSB_PATH/lib/pkgconfig";
else
eval "$(/usr/local/Homebrew/bin/brew shellenv)";
fi
rm -rf "$BUILD_DIRECTORY";
mkdir "$BUILD_DIRECTORY";
cd "$BUILD_DIRECTORY";
qmake \
-spec macx-clang \
CONFIG+="release qtquickcompiler" \
-o Makefile \
../$PROJECT.pro \
QMAKE_APPLE_DEVICE_ARCHS="x86_64 arm64";
make qmake_all;
make "-j$(sysctl -n hw.ncpu)" > /dev/null 2>&1;
make install;
# bundle libusb
mkdir -p "$PROJECT.app/Contents/Frameworks";
cp "$LIBUSB_PATH/lib/libusb-1.0.0.dylib" "$PROJECT.app/Contents/Frameworks";
relink_framework()
{
local FILE;
local LIB;
local REL_PATH;
FILE="$1";
LIB="$2";
REL_PATH="$3";
PATHS=( $(otool -L "$FILE" | grep "$LIB" | awk '{print $1}' ) );
for CUR in "${PATHS[@]}"; do
install_name_tool -change "$CUR" "$REL_PATH" "$FILE";
done
}
relink_framework \
"$PROJECT.app/Contents/Frameworks/libusb-1.0.0.dylib" \
"libusb-1.0.0.dylib" \
"@loader_path/libusb-1.0.0.dylib";
relink_framework \
"$PROJECT.app/Contents/MacOS/qFlipper" \
"libusb-1.0.0.dylib" \
"@loader_path/../Frameworks/libusb-1.0.0.dylib";
relink_framework \
"$PROJECT.app/Contents/MacOS/qFlipper-cli" \
"libusb-1.0.0.dylib" \
"@loader_path/../Frameworks/libusb-1.0.0.dylib";
# Sign
if [ -n "$MAC_OS_SIGNING_KEY_ID" ]
then
xattr -cr "$PROJECT.app"
codesign --force --options=runtime -s "$MAC_OS_SIGNING_KEY_ID" --deep -v "$PROJECT.app"
/usr/bin/ditto -c -k --keepParent "$PROJECT.app" "$PROJECT.zip"
if [ -n "${MAC_OS_SIGNING_KEY_ID:-""}" ]; then
security default-keychain -s "$MAC_OS_KEYCHAIN_NAME";
security unlock-keychain -p "$MAC_OS_KEYCHAIN_PASSWORD" "$MAC_OS_KEYCHAIN_NAME";
xattr -cr "$PROJECT.app";
codesign --force --options=runtime -s "$MAC_OS_SIGNING_KEY_ID" --deep -v "$PROJECT.app";
/usr/bin/ditto -c -k --keepParent "$PROJECT.app" "$PROJECT.zip";
xcrun altool \
--notarize-app \
--primary-bundle-id "$MAC_OS_SIGNING_BUNDLE_ID" \
--username "$MAC_OS_SIGNING_USERNAME" \
--password "$MAC_OS_SIGNING_PASSWORD" \
--asc-provider $MAC_OS_SIGNING_ASC_PROVIDER \
--file "$PROJECT.zip"
--asc-provider "$MAC_OS_SIGNING_ASC_PROVIDER" \
--file "$PROJECT.zip";
fi
# build DMG
mkdir disk_image
mv $PROJECT.app disk_image
create-dmg \
mkdir disk_image;
mv "$PROJECT.app" "disk_image/";
cp "../installer-assets/macos/DS_Store" "disk_image/.DS_Store";
cp "../installer-assets/macos/VolumeIcon.icns" "disk_image/.VolumeIcon.icns";
cp -r "../installer-assets/macos/background" "disk_image/.background";
../scripts/create-dmg/create-dmg \
--volname "$PROJECT-$(git describe --tags --abbrev=0)" \
--volicon "../installer-assets/icons/${PROJECT}-installer.icns" \
--background "../installer-assets/backgrounds/qFlipper_disk_background.png" \
--window-pos 200 120 \
--window-size 600 400 \
--icon-size 100 \
--icon "$PROJECT.app" 125 150 \
--hide-extension "$PROJECT.app" \
--skip-jenkins \
--app-drop-link 485 150 \
"$PROJECT.dmg" \
"disk_image/"
"disk_image/";

View File

@ -11,22 +11,14 @@ CONFIG -= app_bundle
unix|win32 {
LIBS += \
-L$$OUT_PWD/../backend/ -lbackend \
-L$$OUT_PWD/../3rdparty/ -l3rdparty \
-L$$OUT_PWD/../plugins/ -lflipperproto0 \
-L$$OUT_PWD/../backend/ -lbackend \
-L$$OUT_PWD/../dfu/ -ldfu
}
win32:!win32-g++ {
PRE_TARGETDEPS += \
$$OUT_PWD/../backend/backend.lib \
$$OUT_PWD/../3rdparty/3rdparty.lib \
$$OUT_PWD/../dfu/dfu.lib
} else:unix|win32-g++ {
PRE_TARGETDEPS += \
$$OUT_PWD/../backend/libbackend.a \
$$OUT_PWD/../3rdparty/lib3rdparty.a \
$$OUT_PWD/../dfu/libdfu.a
contains(CONFIG, static): LIBS += \
-L$$OUT_PWD/../3rdparty/ -l3rdparty \
-L$$OUT_PWD/../plugins/ -lflipperproto0
}
win32 {
@ -51,11 +43,6 @@ INCLUDEPATH += \
$$PWD/../dfu \
$$PWD/../backend
DEPENDPATH += \
$$PWD/../dfu \
$$PWD/../backend \
$$PWD/../3rdparty \
SOURCES += \
main.cpp \
cli.cpp

Binary file not shown.

Binary file not shown.

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.9 KiB

View File

@ -62,13 +62,6 @@ unix|win32 {
LIBS += -L$$OUT_PWD/../../3rdparty/ -l3rdparty
}
win32:!win32-g++ {
PRE_TARGETDEPS += $$OUT_PWD/../../3rdparty/3rdparty.lib
} else:unix|win32-g++ {
PRE_TARGETDEPS += $$OUT_PWD/../../3rdparty/lib3rdparty.a
}
DEFINES += PB_ENABLE_MALLOC
unix:!macx {

View File

@ -16,6 +16,11 @@ ProtobufPlugin::ProtobufPlugin(QObject *parent):
m_versionMinor(0)
{}
int ProtobufPlugin::versionMajor() const
{
return 0;
}
void ProtobufPlugin::setMinorVersion(int version)
{
m_versionMinor = version;

View File

@ -14,6 +14,7 @@ class ProtobufPlugin : public QObject, public ProtobufPluginInterface
public:
ProtobufPlugin(QObject *parent = nullptr);
int versionMajor() const override;
void setMinorVersion(int version) override;
const QByteArray statusPing(uint32_t id, const QByteArray &data) const override;

View File

@ -16,6 +16,7 @@ public:
virtual ~ProtobufPluginInterface() {}
virtual int versionMajor() const = 0;
virtual void setMinorVersion(int version) = 0;
virtual const QByteArray statusPing(uint32_t id, const QByteArray &data = QByteArray()) const = 0;

View File

@ -19,7 +19,7 @@ unix:!macx {
} else:macx {
DEFINES += USB_BACKEND_LIBUSB
PKG_CONFIG = /usr/local/bin/pkg-config
PKG_CONFIG = /opt/homebrew/bin/pkg-config
CONFIG += link_pkgconfig
PKGCONFIG += libusb-1.0 zlib

30
scripts/build-libusb.sh Executable file
View File

@ -0,0 +1,30 @@
#!/bin/bash
TMP_DIR="/tmp/$(openssl rand -hex 3)";
LIBUSB_PATH="/opt/libs/libusb/1.0.24-universal";
build()
{
local ARCH="$1";
make clean;
rm -rf "$TMP_DIR/libusb-$ARCH";
rm -f Makefile;
arch -"$ARCH" ./configure --prefix=/Users/build/temp/libusb-"$ARCH" CFLAGS="-mmacosx-version-min=10.14";
arch -"$ARCH" make;
arch -"$ARCH" make install;
}
join()
{
lipo \
-create \
-output \
"$LIBUSB_PATH/lib/libusb-1.0.0.dylib" \
"$TMP_DIR/libusb-x86_64/lib/libusb-1.0.0.dylib" \
"$TMP_DIR/libusb-arm64/lib/libusb-1.0.0.dylib";
}
build "x86_64";
build "arm64";
join;
rm -rf "$TMP_DIR";

31
scripts/build-static-qt.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/bash
QT_VERSION="6.3.1";
QT_URL="https://download.qt.io/official_releases/qt/6.3/$QT_VERSION/single/qt-everywhere-src-$QT_VERSION.tar.xz"
QT_INSTALL_DIR="/opt/homebrew/qt-6.3.1-static"
rm -rf "qt-everywhere-src-$QT_VERSION";
wget "$QT_URL";
tar -xvf "qt-everywhere-src-$QT_VERSION.tar.zx";
cd "qt-everywhere-src-$QT_VERSION";
mkdir "build";
cd "build";
/opt/homebrew/Cellar/cmake/3.24.1/bin/cmake \
'-DBUILD_qtwebengine=OFF' \
'-DBUILD_SHARED_LIBS=OFF' \
'-DCMAKE_INSTALL_PREFIX=/opt/homebrew/qt-6.3.1-static' \
'-DQT_BUILD_TESTS=FALSE' \
'-DQT_BUILD_EXAMPLES=FALSE' \
'-DCMAKE_BUILD_TYPE=Release' \
'-DCMAKE_OSX_ARCHITECTURES=x86_64;arm64;' \
'-DQT_BUILD_TESTS_BY_DEFAULT=OFF' \
'-DINPUT_sql_mysql=no' \
'-DINPUT_sql_odbc=no' \
'-DINPUT_sql_psql=no' \
'-DINPUT_system_sqlite=no' \
'-G' 'Ninja' \
'/Users/build/qt-everywhere-src-6.3.1'
/opt/homebrew/Cellar/cmake/3.24.1/bin/cmake --build . --parallel
/opt/homebrew/Cellar/cmake/3.24.1/bin/cmake --install .

1
scripts/create-dmg Submodule

@ -0,0 +1 @@
Subproject commit 11cff56757861851cdbec5c20c9493fc6f87a32f