mii/AzooKeyKanaKanjiConverter
1
0
Fork 0
mirror of https://github.com/mii443/AzooKeyKanaKanjiConverter.git synced 2025-08-22 15:05:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: zenz-v1の出力を学習の適用次に考慮するよう変更

Browse Source
This commit is contained in:
Miwa / Ensan
2024-07-03 01:21:59 +09:00
parent c6cde79e5c
commit 33339216c4
1 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Sources/KanaKanjiConverterModule/Zenz/ZenzContext.swift
View File

@ -168,6 +168,8 @@ class ZenzContext {
var exp_sum: Float = 0
var max_token: llama_token = 0
var max_exp: Float = .infinity * -1
var actual_token = token_id
var actual_exp: Float = .infinity * -1
let startIndex = (i - 1 - startOffset) * Int(n_vocab)
let endIndex = (i - startOffset) * Int(n_vocab)
for index in startIndex ..< endIndex {
@ -177,6 +179,9 @@ class ZenzContext {
max_exp = v
max_token = llama_token(index - startIndex)
}
if index == actual_token {
actual_exp = v
}
}
// ここで最も良い候補であったかをチェックする
if max_token != token_id {
@ -195,8 +200,11 @@ class ZenzContext {
$0.append(contentsOf: token_to_piece(token: $1))
}
let acceptedPrefix = String(cString: cchars + [0]).dropFirst(prompt.count)
if let nextDicdataElement = getNextDicdataElement(for: String(acceptedPrefix), of: candidate), nextDicdataElement.metadata.contains(.isLearned) {
// 学習による候補なので素通しする
if let nextDicdataElement = getNextDicdataElement(for: String(acceptedPrefix), of: candidate),
nextDicdataElement.metadata.contains(.isLearned),
actual_exp * 10 > max_exp
{
// 学習による候補であり、zenzによる評価も許容範囲なので、素通しする
// pass
} else {
// adding "\0"