Merge branch 'develop' into fix/enable_user_dictionary_wo_zenzai

This commit is contained in:
Miwa / Ensan
2024-09-28 17:27:51 +09:00
8 changed files with 21 additions and 14 deletions

View File

@@ -3,7 +3,7 @@ import ArgumentParser
@main
public struct Anco: AsyncParsableCommand {
public static var configuration = CommandConfiguration(
public static let configuration = CommandConfiguration(
abstract: "Anco is A(zooKey) Kana-Ka(n)ji (co)nverter",
subcommands: [Subcommands.Run.self, Subcommands.Dict.self, Subcommands.Evaluate.self, Subcommands.Session.self, Subcommands.ExperimentalPredict.self],
defaultSubcommand: Subcommands.Run.self

View File

@@ -4,7 +4,7 @@ import ArgumentParser
extension Subcommands {
struct Dict: ParsableCommand {
static var configuration = CommandConfiguration(
static let configuration = CommandConfiguration(
commandName: "dict",
abstract: "Show dict information",
subcommands: [Self.Read.self]

View File

@@ -29,7 +29,7 @@ extension Subcommands.Dict {
@Option(name: [.customLong("sort")], help: "Sort order")
var sortOrder: SortOrder = .ruby
static var configuration = CommandConfiguration(
static let configuration = CommandConfiguration(
commandName: "read",
abstract: "Read dictionary data and extract informations"
)

View File

@@ -3,7 +3,7 @@ import ArgumentParser
import Foundation
extension Subcommands {
struct Evaluate: ParsableCommand {
struct Evaluate: AsyncParsableCommand {
@Argument(help: "query, answer, tagを備えたjsonファイルへのパス")
var inputFile: String = ""
@@ -18,7 +18,7 @@ extension Subcommands {
@Option(name: [.customLong("config_zenzai_inference_limit")], help: "inference limit for zenzai.")
var configZenzaiInferenceLimit: Int = .max
static var configuration = CommandConfiguration(commandName: "evaluate", abstract: "Evaluate quality of Conversion for input data.")
static let configuration = CommandConfiguration(commandName: "evaluate", abstract: "Evaluate quality of Conversion for input data.")
private func parseInputFile() throws -> [InputItem] {
let url = URL(fileURLWithPath: self.inputFile)
@@ -26,15 +26,15 @@ extension Subcommands {
return try JSONDecoder().decode([InputItem].self, from: data)
}
@MainActor mutating func run() throws {
mutating func run() async throws {
let inputItems = try parseInputFile()
let requestOptions = requestOptions()
let converter = KanaKanjiConverter()
let converter = await KanaKanjiConverter()
let start = Date()
var resultItems: [EvaluateItem] = []
for item in inputItems {
//
converter.sendToDicdataStore(.importDynamicUserDict(
await converter.sendToDicdataStore(.importDynamicUserDict(
(item.user_dictionary ?? []).map {
DicdataElement(word: $0.word, ruby: $0.reading.toKatakana(), cid: CIDData..cid, mid: MIDData..mid, value: -10)
}
@@ -43,7 +43,7 @@ extension Subcommands {
var composingText = ComposingText()
composingText.insertAtCursorPosition(item.query, inputStyle: .direct)
let result = converter.requestCandidates(composingText, options: requestOptions)
let result = await converter.requestCandidates(composingText, options: requestOptions)
let mainResults = result.mainResults.filter {
$0.data.reduce(into: "", {$0.append(contentsOf: $1.ruby)}) == item.query.toKatakana()
}
@@ -57,7 +57,7 @@ extension Subcommands {
)
)
// Explictly reset state
converter.stopComposition()
await converter.stopComposition()
}
let end = Date()
var result = EvaluateResult(n_best: self.configNBest, execution_time: end.timeIntervalSince(start), items: resultItems)

View File

@@ -12,7 +12,7 @@ extension Subcommands {
@Option(name: [.customLong("zenz")], help: "gguf format model weight for zenz.")
var zenzWeightPath: String = ""
static var configuration = CommandConfiguration(commandName: "experimental_predict", abstract: "Show help for this utility.")
static let configuration = CommandConfiguration(commandName: "experimental_predict", abstract: "Show help for this utility.")
@MainActor mutating func run() async {
let converter = KanaKanjiConverter()

View File

@@ -25,7 +25,7 @@ extension Subcommands {
@Flag(name: [.customLong("report_score")], help: "Show internal score for the candidate.")
var reportScore = false
static var configuration = CommandConfiguration(commandName: "run", abstract: "Show help for this utility.")
static let configuration = CommandConfiguration(commandName: "run", abstract: "Show help for this utility.")
@MainActor mutating func run() async {
let converter = KanaKanjiConverter()

View File

@@ -32,7 +32,7 @@ extension Subcommands {
@Flag(name: [.customLong("zenz_v1")], help: "Use zenz_v1 model.")
var zenzV1 = false
static var configuration = CommandConfiguration(commandName: "session", abstract: "Start session for incremental input.")
static let configuration = CommandConfiguration(commandName: "session", abstract: "Start session for incremental input.")
private func getTemporaryDirectory() -> URL? {
let fileManager = FileManager.default

View File

@@ -227,7 +227,8 @@ public struct ComposingText: Sendable {
return
}
let n_prefix = self.input[0 ..< inputCursorPosition].suffix {$0.character == "n" && $0.inputStyle == .roman2kana}
if n_prefix.count % 2 == 1 && !["n", "a", "i", "u", "e", "o", "y"].contains(first) {
if n_prefix.count % 2 == 1 && !["n", "a", "i", "u", "e", "o", "y"].contains(first)
&& self.input.dropLast(n_prefix.count).last != .init(character: "x", inputStyle: .roman2kana) {
self.input[inputCursorPosition - 1] = InputElement(character: "", inputStyle: .direct)
self.input.insert(contentsOf: string.map {InputElement(character: $0, inputStyle: inputStyle)}, at: inputCursorPosition)
return
@@ -428,12 +429,18 @@ extension ComposingText {
return true
}
let n_suffix = originalElements[0 ..< leftIndex].suffix(while: {$0.inputStyle == .roman2kana && $0.character == "n"})
// nnvalid
if n_suffix.count % 2 == 0 && !n_suffix.isEmpty {
return true
}
// nny-nnvalid
if n_suffix.count % 2 == 1 && !["a", "i", "u", "e", "o", "y", "n"].contains(firstElement.character) {
return true
}
// n1xvalid (xn)
if n_suffix.count % 2 == 1 && originalElements.dropLast(n_suffix.count).last == .init(character: "x", inputStyle: .roman2kana) {
return true
}
return false
}