CI: Androidのビルドサポートを追加 (#142)

* ci: change swift version to stable * ci: change trigger for test * revert changes * ci: fix version * ci: change display text from branch to tag * a * ci * fix trigger * fix * fix for android (use float func) * ci: heckout with submodules * ci: remove unneccesary files * ふぃふぃふぃ * ci: copy neccesary files * ci: fix for arm build * ci: arm * nn> * a * tree * cp -r * haaa? * ci: 大文字と小文字 * ci: re * ci: copy resource folder * ci: change path * ci: default dictionary * ci: こうか？ * ci: revert changes * ci: clean workflow * fix * ｎ * a * ci: incorporate jobs * fix variable * fix path * ci: incorporate android actions to swift.yml * fix * ci: fix yaml syntax error * ci: fix syntax error? * ci: format * ci: use valid yml * ci: use release tag instead of latest tag * ci: cache * ci: build llama on android? * ci: add header include path * a * a * hensu * fix: llama mock * uninclude llama * change os(android) to canimport(andoid) * a * remove linux * a * n * ci: check with env value * fix LLAMA_MOCK * Update .gitignore
2025-08-22 06:55:26 +00:00 · 2024-09-30 22:13:01 +09:00
parent 36a73c135d
commit 96e8f37a01
6 changed files with 316 additions and 55 deletions
--- a/.github/workflows/swift.yml
+++ b/.github/workflows/swift.yml
@ -4,9 +4,9 @@
 name: Swift Build and Test
 on:
  push:
-    branches: [ "main", "develop" ]
+    branches: ["main", "develop"]
  pull_request:
-    branches: [ "main", "develop" ]
+    branches: ["main", "develop"]

 jobs:
  macos-build:
@ -46,19 +46,21 @@ jobs:
      - name: Run tests
        run: swift test -c release -Xswiftc -strict-concurrency=complete -v
  windows-build:
-      name: Swift ${{ matrix.swift-version }} on ${{ matrix.os }}
+    name: Swift ${{ matrix.swift-version.tag }} on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-latest]
-          swift-version: ["20240802.2"]
+        swift-version:
+          [{
+            branch: "swift-6.0.1-release",
+            tag: "6.0.1-RELEASE"
+          }]
    steps:
      - uses: compnerd/gha-setup-swift@main
        with:
-          release-tag-name: ${{ matrix.swift-version }}
-          github-repo: thebrowsercompany/swift-build
-          release-asset-name: installer-amd64.exe
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ matrix.swift-version.branch }}
+          tag: ${{ matrix.swift-version.tag }}
      - uses: actions/checkout@v4
        with:
          submodules: true
@ -78,4 +80,185 @@ jobs:
        run: swift build -Xswiftc -strict-concurrency=complete -v
      - name: Run tests
        run: swift test -c release -Xswiftc -strict-concurrency=complete -v
+  android-build:
+    # from: finagolfin/swift-android-sdk
+    name: Swift on android
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [x86_64, aarch64, armv7]
+        os: [ubuntu-latest]

+    env:
+      ANDROID_API_LEVEL: 24
+
+    steps:
+      - name: Check for latest Swift toolchain
+        id: check
+        run: |
+          LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.0 | cut -d- -f2)
+          SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE"
+          echo "release-tag=$SWIFT_TAG" >> $GITHUB_OUTPUT
+
+      - name: Get cached Swift toolchain
+        id: cache-toolchain-ubuntu
+        uses: actions/cache@v4
+        with:
+          path: ~/${{ steps.check.outputs.release-tag }}-ubuntu22.04.tar.gz
+          key: swift-ubuntu-22.04-${{ steps.check.outputs.release-tag }}-toolchain
+          lookup-only: true
+      - name: Download toolchain
+        id: download
+        if: ${{ steps.cache-toolchain-ubuntu.outputs.cache-hit != 'true' }}
+        run: |
+          SWIFT_TAG="${{ steps.check.outputs.release-tag }}"
+          SWIFT_BRANCH="swift-$(echo ${{ steps.check.outputs.release-tag }} | cut -d- -f2)-release"
+          cd
+          wget -q https://download.swift.org/$SWIFT_BRANCH/ubuntu2204/$SWIFT_TAG/$SWIFT_TAG-ubuntu22.04.tar.gz
+          echo "downloaded latest Ubuntu toolchain: ${SWIFT_TAG}"
+
+      - name: Get cached Android SDK
+        id: cache-sdk
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/swift-release-android-${{ matrix.arch }}-*-sdk.tar.xz
+          key: ${{ steps.check.outputs.release-tag }}-${{ matrix.arch }}-ndk-27b-sdk
+
+      - name: Clone Android SDK config
+        uses: actions/checkout@v4
+        with:
+          repository: finagolfin/swift-android-sdk
+          path: sdk-config
+      - name: Get cached ${{ steps.check.outputs.release-tag }} toolchain
+        if: ${{ steps.cache-toolchain-ubuntu.outputs.cache-hit == 'true' }}
+        id: cache-toolchain
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/${{ steps.check.outputs.release-tag }}-ubuntu22.04.tar.gz
+          key: swift-ubuntu-22.04-${{ steps.check.outputs.release-tag }}-toolchain
+          fail-on-cache-miss: true
+      - name: Setup Swift Toolchain
+        run: |
+          set -x
+          cd sdk-config
+          tar xf ~/${{ steps.check.outputs.release-tag }}-ubuntu22.04.tar.gz
+          TOOLCHAIN=${PWD}/${{ steps.check.outputs.release-tag }}-ubuntu22.04/usr
+          perl -pi -e 's%canImport\(Bionic%canImport\(Android%' $TOOLCHAIN/bin/swift-package
+          perl -pi -e 's%import Bionic%import Android%' $TOOLCHAIN/bin/swift-package
+          perl -pi -e 's%TSCBasic, would be%TSCBasic, would %' $TOOLCHAIN/bin/swift-package
+          echo "TOOLCHAIN=${TOOLCHAIN}" >> $GITHUB_ENV
+          ${TOOLCHAIN}/bin/swift --version
+
+      - name: Build Swift release Android SDK
+        # build-script currently only works on ubuntu
+        if: ${{ (steps.cache-sdk.outputs.cache-hit != 'true') }}
+        env:
+          SWIFT_TAG: ${{ steps.check.outputs.release-tag }}
+          ANDROID_ARCH: ${{ matrix.arch }}
+        run: |
+          set -x
+          sudo apt install ninja-build
+          cd sdk-config
+          ${TOOLCHAIN}/bin/swift --version
+          BUILD_SWIFT_PM=1 ${TOOLCHAIN}/bin/swift get-packages-and-swift-source.swift
+          SDK_NAME=$(ls | grep swift-release-android-${{ matrix.arch }})
+          SDK=`pwd`/$SDK_NAME
+
+          git apply swift-android.patch swift-android-foundation.patch swift-android-ci.patch swift-crypto.patch swift-system.patch
+          git apply swift-android-ci-except-trunk.patch swift-android-foundation-except-trunk.patch
+          git apply -C1 swift-android-foundation-armv7-except-trunk.patch
+
+          perl -pi -e 's%r26%ndk/27%' swift/stdlib/cmake/modules/AddSwiftStdlib.cmake
+          perl -pi -e "s%/data/data/com.termux/files%$SDK%g" $SDK/usr/lib/pkgconfig/sqlite3.pc
+          perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift
+          ./swift/utils/build-script -RA --skip-build-cmark --build-llvm=0 --android --android-ndk $ANDROID_NDK_LATEST_HOME --android-arch ${{ matrix.arch }} --android-api-level $ANDROID_API_LEVEL --build-swift-tools=0 --native-swift-tools-path=${TOOLCHAIN}/bin --native-clang-tools-path=${TOOLCHAIN}/bin --cross-compile-hosts=android-${{ matrix.arch }} --cross-compile-deps-path=$SDK --skip-local-build --build-swift-static-stdlib --xctest --skip-early-swift-driver --install-swift --install-libdispatch --install-foundation --install-xctest --install-destdir=$SDK --swift-install-components='clang-resource-dir-symlink;license;stdlib;sdk-overlay' --cross-compile-append-host-target-to-destdir=False -b -p --install-llbuild --sourcekit-lsp --skip-early-swiftsyntax
+
+          cp $ANDROID_NDK_LATEST_HOME/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/$(echo ${{ matrix.arch }} | sed "s/v7//")-linux-android*/libc++_shared.so $SDK/usr/lib
+          patchelf --set-rpath \$ORIGIN $SDK/usr/lib/swift/android/libdispatch.so
+          patchelf --set-rpath \$ORIGIN/../..:\$ORIGIN $SDK/usr/lib/swift/android/lib[FXs]*.so
+          tar cJf ~/$SDK_NAME.tar.xz $SDK_NAME
+          rm -rf build/ $SDK_NAME llvm-project/
+
+      - name: Cache new SDK
+        if: ${{ (steps.cache-sdk.outputs.cache-hit != 'true') }}
+        uses: actions/cache/save@v4
+        with:
+          path: ~/swift-release-android-${{ matrix.arch }}-*-sdk.tar.xz
+          key: ${{ steps.check.outputs.release-tag }}-${{ matrix.arch }}-ndk-27b-sdk
+      - name: Setup Swift release Android SDK
+        id: sdk-setup
+        run: |
+          set -x
+          cd sdk-config
+          tar xf ~/swift-release-android-${{ matrix.arch }}*-sdk.tar.xz
+          pushd swift-*-sdk
+          SDK_PATH=${PWD}
+          popd
+
+          ARCH_JSON=${SDK_PATH}/usr/swiftpm-android-${{ matrix.arch }}.json
+          NDK_PREBUILT=${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64
+
+          cat ${ARCH_JSON}
+
+          echo "SDK_PATH=${SDK_PATH}" >> $GITHUB_ENV
+          echo "ARCH_JSON=${ARCH_JSON}" >> $GITHUB_ENV
+      - uses: actions/checkout@v4
+        with:
+          path: AzooKeyKanaKanjiConverter
+          submodules: true
+      - name: Build
+        run: |
+          cd AzooKeyKanaKanjiConverter
+          LLAMA_MOCK=1 ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android
+      - name: Get cached Termux app
+        if: ${{ matrix.arch == 'x86_64' }}
+        id: cache-termux
+        uses: actions/cache@v4
+        with:
+          path: ~/termux-debug.apk
+          key: termux-app
+      - name: Get Termux debug app if not cached
+        if: ${{ (matrix.arch == 'x86_64') && (steps.cache-termux.outputs.cache-hit != 'true') }}
+        run: wget -O ~/termux-debug.apk https://github.com/termux/termux-app/releases/download/v0.118.0/termux-app_v0.118.0+github-debug_x86_64.apk
+      - name: Prepare Android emulator
+        if: ${{ matrix.arch == 'x86_64' }}
+        run: |
+          set -x
+          # create the test runner script
+          cat > ~/test-toolchain.sh << EOF
+          adb install ~/termux-debug.apk
+          adb push pack /data/local/tmp
+          EOF
+
+          mkdir -p pack/lib/swift/android
+          TARGET="x86_64-unknown-linux-android$ANDROID_API_LEVEL"
+
+          cp AzooKeyKanaKanjiConverter/.build/$TARGET/debug/AzooKeyKanakanjiConverterPackageTests.xctest pack
+          echo 'adb shell /data/local/tmp/pack/AzooKeyKanakanjiConverterPackageTests.xctest' >> ~/test-toolchain.sh
+
+          cp -r AzooKeyKanaKanjiConverter/.build/$TARGET/debug/AzooKeyKanakanjiConverter_KanaKanjiConverterModuleTests.resources pack
+          cp -r AzooKeyKanaKanjiConverter/.build/$TARGET/debug/AzooKeyKanakanjiConverter_KanaKanjiConverterModuleWithDefaultDictionary.resources pack
+
+          cp sdk-config/swift-*-android-x86_64-*${ANDROID_API_LEVEL}-sdk/usr/lib/lib*so pack/lib
+          cp sdk-config/swift-*-android-x86_64-*${ANDROID_API_LEVEL}-sdk/usr/lib/swift/android/lib*so pack/lib/swift/android
+          mv pack/lib/libc++_shared.so pack/lib/swift/android
+          # need to free up some space or else the emulator fails to launch:
+          # ERROR   | Not enough space to create userdata partition. Available: 6086.191406 MB at /home/runner/.android/avd/../avd/test.avd, need 7372.800000 MB.
+          rm -rf sdk-config/${{ steps.check.outputs.release-tag }}-ubuntu22.04 */.build
+
+          chmod +x ~/test-toolchain.sh
+
+          echo "TEST SCRIPT:"
+          cat ~/test-toolchain.sh
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      - name: Run tests
+        if: ${{ matrix.arch == 'x86_64' }}
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: 29
+          arch: x86_64
+          script: ~/test-toolchain.sh
--- a/.gitignore
+++ b/.gitignore
@ -18,3 +18,5 @@ Package.resolved
 .docc-build
 .vscode
 *.gguf
+*.dll
+*.lib
--- a/Package.swift
+++ b/Package.swift
@ -2,6 +2,7 @@
 // The swift-tools-version declares the minimum version of Swift required to build this package.

 import PackageDescription
+import Foundation

 let swiftSettings: [SwiftSetting] = [
    .enableUpcomingFeature("BareSlashRegexLiterals"),
@ -79,13 +80,6 @@ var targets: [Target] = [
 ]


-#if !(os(Windows))
-dependencies.append(
-    .package(url: "https://github.com/ensan-hcl/llama.cpp", branch: "6b862f4")
-)
-#endif
-
-
 #if os(Windows)
 targets.append(contentsOf: [
    .systemLibrary(
@ -102,7 +96,25 @@ targets.append(contentsOf: [
    )
 ])
 #else
-targets.append(contentsOf: [
+if let envValue = ProcessInfo.processInfo.environment["LLAMA_MOCK"], envValue == "1" {
+    targets.append(contentsOf: [
+        .target(name: "llama-mock"),
+        .target(
+            name: "KanaKanjiConverterModule",
+            dependencies: [
+                "SwiftUtils",
+                "llama-mock",
+                .product(name: "Collections", package: "swift-collections")
+            ],
+            swiftSettings: swiftSettings
+        )
+    ])
+} else {
+    dependencies.append(
+        .package(url: "https://github.com/ensan-hcl/llama.cpp", branch: "6b862f4")
+    )
+
+    targets.append(contentsOf: [
        .target(
            name: "KanaKanjiConverterModule",
            dependencies: [
@ -112,7 +124,8 @@ targets.append(contentsOf: [
            ],
            swiftSettings: swiftSettings
        )
-])
+    ])
+}
 #endif

 let package = Package(
--- a/Sources/KanaKanjiConverterModule/DicdataStore/DicdataStore.swift
+++ b/Sources/KanaKanjiConverterModule/DicdataStore/DicdataStore.swift
@ -579,7 +579,7 @@ public final class DicdataStore {
        result.append(contentsOf: self.getJapaneseNumberDicdata(head: convertTarget))
        if inputData.input[..<inputRange.startIndex].last?.character.isNumber != true && inputData.input[inputRange.endIndex...].first?.character.isNumber != true, let number = Int(convertTarget) {
            result.append(DicdataElement(ruby: convertTarget, cid: CIDData.数.cid, mid: MIDData.小さい数字.mid, value: -14))
-            if number <= Int(1E12) && -Int(1E12) <= number, let kansuji = self.numberFormatter.string(from: NSNumber(value: number)) {
+            if Double(number) <= 1E12 && -1E12 <= Double(number), let kansuji = self.numberFormatter.string(from: NSNumber(value: number)) {
                result.append(DicdataElement(word: kansuji, ruby: convertTarget, cid: CIDData.数.cid, mid: MIDData.小さい数字.mid, value: -16))
            }
        }
--- a/Sources/KanaKanjiConverterModule/Zenz/ZenzContext.swift
+++ b/Sources/KanaKanjiConverterModule/Zenz/ZenzContext.swift
@ -1,4 +1,8 @@
+#if canImport(llama)
 import llama
+#else
+import llama_mock
+#endif
 import SwiftUtils
 import HeapModule
 import Algorithms
@ -160,9 +164,9 @@ class ZenzContext {
            // FIXME: there can be more efficient implementations, poossibly using Accelerate or other frameworks.
            var log_prob: Float = 0
            for index in ((i - 1) * Int(n_vocab)) ..< (i * Int(n_vocab)) {
-                log_prob += exp(logits[index])
+                log_prob += expf(logits[index])
            }
-            log_prob = log(log_prob)
+            log_prob = logf(log_prob)
            log_prob = logits[Int((i - 1) * Int(n_vocab) + Int(token_id))] - log_prob
            sum += log_prob
        }
@ -228,7 +232,7 @@ class ZenzContext {
        for index in startIndex..<endIndex {
            let token = llama_token(index - startIndex)
            let repeat_penalty = Float(1.0 + token_to_penalty_weight[token, default: 0])
-            let v = exp(logits[index] / repeat_penalty)
+            let v = expf(logits[index] / repeat_penalty)
            exp_sum += v

            let tokenPieceData = Data((token_to_piece(token: token)).map(UInt8.init))
@ -332,7 +336,7 @@ class ZenzContext {
            let endIndex = (i - startOffset) * Int(n_vocab)
            var tokenHeap = FixedSizeHeap<TokenAndExpLogit>(size: requestRichCandidates ? 3 : 1)
            for index in startIndex ..< endIndex {
-                let v = exp(logits[index])
+                let v = expf(logits[index])
                exp_sum += v
                tokenHeap.insertIfPossible(TokenAndExpLogit(token: llama_token(index - startIndex), expLogit: v))
            }
@ -353,7 +357,7 @@ class ZenzContext {
                    let wholeResult = String(string.dropFirst(prompt.count))
                    return .wholeResult(wholeResult)
                } else {
-                    let actual_exp: Float = exp(logits[startIndex + Int(token_id)])
+                    let actual_exp: Float = expf(logits[startIndex + Int(token_id)])
                    // 学習されたトークンであり、なおかつactual_expのある程度大きければ、学習されたトークンを優先する
                    let preferLearnedToken = is_learned_token[i].isLearned && actual_exp * is_learned_token[i].priority > maxItem.expLogit
                    if !preferLearnedToken {
@ -380,7 +384,7 @@ class ZenzContext {
                    )
                }
            }
-            score += log(maxItem.expLogit) - log(exp_sum)
+            score += logf(maxItem.expLogit) - logf(exp_sum)
        }
        return .pass(score: score, alternativeConstraints: altTokens.unordered.sorted(by: >).map {.init(probabilityRatio: $0.probabilityRatioToMaxProb, prefixConstraint: $0.constraint)})
    }
--- a/Sources/llama-mock/llama-mock.swift
+++ b/Sources/llama-mock/llama-mock.swift
@ -0,0 +1,59 @@
+private func unimplemented<T>() -> T {
+    fatalError("unimplemented")
+}
+
+package typealias llama_token = Int32
+package typealias llama_pos = Int32
+package typealias llama_seq_id = Int32
+
+package struct llama_context_params {
+    package var seed: Int
+    package var n_ctx: Int
+    package var n_threads: UInt32
+    package var n_threads_batch: UInt32
+    package var n_batch: Int
+}
+package func llama_context_default_params() -> llama_context_params { unimplemented() }
+
+package typealias llama_context = OpaquePointer
+package func llama_new_context_with_model(_ model: llama_model, _ ctx_params: llama_context_params) -> llama_context? { unimplemented() }
+package func llama_free(_ context: llama_context) {}
+
+package typealias llama_model = OpaquePointer
+
+package func llama_free_model(_ model: llama_model) {}
+
+package func llama_backend_init() {}
+package func llama_backend_free() {}
+
+package struct llama_model_params {
+    package var use_mmap: Bool
+}
+package func llama_model_default_params() -> llama_model_params { unimplemented() }
+
+package func llama_load_model_from_file(_ path: String, _ model_params: llama_model_params) -> llama_model? { unimplemented() }
+
+package func llama_kv_cache_seq_rm(_ ctx: llama_context, _ seq_id: llama_seq_id, _ p0: llama_pos, _ p1: llama_pos) {}
+package func llama_kv_cache_seq_pos_max(_ ctx: llama_context, _ seq_id: llama_seq_id) -> Int { unimplemented() }
+
+package struct llama_batch {
+    package var token: [llama_token]
+    package var pos: [llama_pos]
+    package var n_seq_id: [llama_seq_id]
+    package var seq_id: [[llama_seq_id]?]
+    package var logits: UnsafeMutablePointer<Float>
+    package var n_tokens: Int
+
+}
+package func llama_batch_init(_ n_tokens: Int, _ embd: Int, _ n_seq_max: Int) -> llama_batch { unimplemented() }
+
+package func llama_n_ctx(_ ctx: llama_context) -> Int { unimplemented() }
+package func llama_n_vocab(_ model: llama_model) -> Int { unimplemented() }
+
+package func llama_tokenize(_ model: llama_model, _ text: String, _ text_len: Int32, _ tokens: UnsafeMutablePointer<llama_token>, _ n_tokens_max: Int32, _ add_special: Bool, _ parse_special: Bool) -> Int { unimplemented() }
+package func llama_token_bos(_ model: llama_model) -> llama_token { unimplemented() }
+package func llama_token_eos(_ model: llama_model) -> llama_token { unimplemented() }
+package func llama_token_to_piece(_ model: llama_model, _ token: llama_token, _ buf: UnsafeMutablePointer<Int8>, _ length: Int32, _ special: Bool) -> Int32 { unimplemented() }
+
+package func llama_decode(_ ctx: llama_context, _ batch: llama_batch) -> Int { unimplemented() }
+package func llama_get_logits(_ ctx: llama_context) -> UnsafeMutablePointer<Float>? { unimplemented() }