mirror of
https://github.com/mii443/mozc.git
synced 2025-08-22 16:15:46 +00:00
Move HistoryReconstructor to a separate file.
#codehealth PiperOrigin-RevId: 700647116
This commit is contained in:
@ -423,6 +423,7 @@ mozc_cc_library(
|
||||
],
|
||||
deps = [
|
||||
":converter_interface",
|
||||
":history_reconstructor",
|
||||
":immutable_converter_interface",
|
||||
":segments",
|
||||
"//base:japanese_util",
|
||||
@ -522,6 +523,40 @@ mozc_cc_test(
|
||||
],
|
||||
)
|
||||
|
||||
mozc_cc_library(
|
||||
name = "history_reconstructor",
|
||||
srcs = ["history_reconstructor.cc"],
|
||||
hdrs = ["history_reconstructor.h"],
|
||||
deps = [
|
||||
":segments",
|
||||
"//base:japanese_util",
|
||||
"//base:util",
|
||||
"//dictionary:pos_matcher",
|
||||
"//protocol:commands_cc_proto",
|
||||
"//testing:friend_test",
|
||||
"@com_google_absl//absl/base:core_headers",
|
||||
"@com_google_absl//absl/strings:string_view",
|
||||
],
|
||||
)
|
||||
|
||||
mozc_cc_test(
|
||||
name = "history_reconstructor_test",
|
||||
srcs = ["history_reconstructor_test.cc"],
|
||||
deps = [
|
||||
":history_reconstructor",
|
||||
":segments",
|
||||
"//data_manager/testing:mock_data_manager",
|
||||
"//dictionary:pos_matcher",
|
||||
"//prediction:dictionary_predictor",
|
||||
"//prediction:predictor",
|
||||
"//prediction:user_history_predictor",
|
||||
"//protocol:commands_cc_proto",
|
||||
"//protocol:config_cc_proto",
|
||||
"//protocol:user_dictionary_storage_cc_proto",
|
||||
"//testing:gunit_main",
|
||||
],
|
||||
)
|
||||
|
||||
mozc_cc_library(
|
||||
name = "pos_id_printer",
|
||||
srcs = ["pos_id_printer.cc"],
|
||||
|
@ -42,11 +42,11 @@
|
||||
#include "absl/log/log.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/types/span.h"
|
||||
#include "base/japanese_util.h"
|
||||
#include "base/strings/assign.h"
|
||||
#include "base/util.h"
|
||||
#include "base/vlog.h"
|
||||
#include "composer/composer.h"
|
||||
#include "converter/history_reconstructor.h"
|
||||
#include "converter/immutable_converter_interface.h"
|
||||
#include "converter/segments.h"
|
||||
#include "dictionary/pos_matcher.h"
|
||||
@ -877,119 +877,4 @@ void Converter::CommitUsageStats(const Segments *segments,
|
||||
UsageStats::IncrementCountBy("SubmittedTotalLength", submitted_total_length);
|
||||
}
|
||||
|
||||
namespace {
|
||||
// Extracts the last substring that consists of the same script type.
|
||||
// Returns true if the last substring is successfully extracted.
|
||||
// Examples:
|
||||
// - "" -> false
|
||||
// - "x " -> "x" / ALPHABET
|
||||
// - "x " -> false
|
||||
// - "C60" -> "60" / NUMBER
|
||||
// - "200x" -> "x" / ALPHABET
|
||||
// (currently only NUMBER and ALPHABET are supported)
|
||||
bool ExtractLastTokenWithScriptType(const absl::string_view text,
|
||||
std::string *last_token,
|
||||
Util::ScriptType *last_script_type) {
|
||||
last_token->clear();
|
||||
*last_script_type = Util::SCRIPT_TYPE_SIZE;
|
||||
|
||||
ConstChar32ReverseIterator iter(text);
|
||||
if (iter.Done()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Allow one whitespace at the end.
|
||||
if (iter.Get() == ' ') {
|
||||
iter.Next();
|
||||
if (iter.Done()) {
|
||||
return false;
|
||||
}
|
||||
if (iter.Get() == ' ') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<char32_t> reverse_last_token;
|
||||
Util::ScriptType last_script_type_found = Util::GetScriptType(iter.Get());
|
||||
for (; !iter.Done(); iter.Next()) {
|
||||
const char32_t codepoint = iter.Get();
|
||||
if ((codepoint == ' ') ||
|
||||
(Util::GetScriptType(codepoint) != last_script_type_found)) {
|
||||
break;
|
||||
}
|
||||
reverse_last_token.push_back(codepoint);
|
||||
}
|
||||
|
||||
*last_script_type = last_script_type_found;
|
||||
// TODO(yukawa): Replace reverse_iterator with const_reverse_iterator when
|
||||
// build failure on Android is fixed.
|
||||
for (std::vector<char32_t>::reverse_iterator it = reverse_last_token.rbegin();
|
||||
it != reverse_last_token.rend(); ++it) {
|
||||
Util::CodepointToUtf8Append(*it, last_token);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace converter {
|
||||
HistoryReconstructor::HistoryReconstructor(
|
||||
const dictionary::PosMatcher &pos_matcher)
|
||||
: pos_matcher_(pos_matcher) {}
|
||||
|
||||
bool HistoryReconstructor::ReconstructHistory(absl::string_view preceding_text,
|
||||
Segments *segments) const {
|
||||
std::string key;
|
||||
std::string value;
|
||||
uint16_t id;
|
||||
if (!GetLastConnectivePart(preceding_text, &key, &value, &id)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Segment *segment = segments->add_segment();
|
||||
segment->set_key(key);
|
||||
segment->set_segment_type(Segment::HISTORY);
|
||||
Segment::Candidate *candidate = segment->push_back_candidate();
|
||||
candidate->rid = id;
|
||||
candidate->lid = id;
|
||||
candidate->content_key = key;
|
||||
candidate->key = std::move(key);
|
||||
candidate->content_value = value;
|
||||
candidate->value = std::move(value);
|
||||
candidate->attributes = Segment::Candidate::NO_LEARNING;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool HistoryReconstructor::GetLastConnectivePart(
|
||||
const absl::string_view preceding_text, std::string *key,
|
||||
std::string *value, uint16_t *id) const {
|
||||
key->clear();
|
||||
value->clear();
|
||||
*id = pos_matcher_.GetGeneralNounId();
|
||||
|
||||
Util::ScriptType last_script_type = Util::SCRIPT_TYPE_SIZE;
|
||||
std::string last_token;
|
||||
if (!ExtractLastTokenWithScriptType(preceding_text, &last_token,
|
||||
&last_script_type)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Currently only NUMBER and ALPHABET are supported.
|
||||
switch (last_script_type) {
|
||||
case Util::NUMBER: {
|
||||
*key = japanese_util::FullWidthAsciiToHalfWidthAscii(last_token);
|
||||
*value = std::move(last_token);
|
||||
*id = pos_matcher_.GetNumberId();
|
||||
return true;
|
||||
}
|
||||
case Util::ALPHABET: {
|
||||
*key = japanese_util::FullWidthAsciiToHalfWidthAscii(last_token);
|
||||
*value = std::move(last_token);
|
||||
*id = pos_matcher_.GetUniqueNounId();
|
||||
return true;
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} // namespace converter
|
||||
} // namespace mozc
|
||||
|
@ -39,6 +39,7 @@
|
||||
'sources': [
|
||||
'<(gen_out_mozc_dir)/dictionary/pos_matcher_impl.inc',
|
||||
'converter.cc',
|
||||
'history_reconstructor.cc',
|
||||
],
|
||||
'dependencies': [
|
||||
'<(mozc_oss_src_dir)/base/absl.gyp:absl_strings',
|
||||
|
@ -40,6 +40,7 @@
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/types/span.h"
|
||||
#include "converter/converter_interface.h"
|
||||
#include "converter/history_reconstructor.h"
|
||||
#include "converter/immutable_converter_interface.h"
|
||||
#include "converter/segments.h"
|
||||
#include "dictionary/pos_matcher.h"
|
||||
@ -51,26 +52,6 @@
|
||||
#include "testing/friend_test.h"
|
||||
|
||||
namespace mozc {
|
||||
namespace converter {
|
||||
class HistoryReconstructor {
|
||||
public:
|
||||
explicit HistoryReconstructor(const dictionary::PosMatcher &pos_matcher);
|
||||
|
||||
ABSL_MUST_USE_RESULT
|
||||
bool ReconstructHistory(absl::string_view preceding_text,
|
||||
Segments *segments) const;
|
||||
|
||||
// Returns the substring of |str|. This substring consists of similar script
|
||||
// type and you can use it as preceding text for conversion.
|
||||
bool GetLastConnectivePart(absl::string_view preceding_text, std::string *key,
|
||||
std::string *value, uint16_t *id) const;
|
||||
|
||||
private:
|
||||
FRIEND_TEST(HistoryReconstructorTest, GetLastConnectivePart);
|
||||
|
||||
const dictionary::PosMatcher &pos_matcher_;
|
||||
};
|
||||
} // namespace converter
|
||||
|
||||
class Converter final : public ConverterInterface {
|
||||
public:
|
||||
|
@ -30,7 +30,6 @@
|
||||
#include "converter/converter.h"
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <iterator>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
@ -1493,75 +1492,6 @@ TEST_F(ConverterTest, StartReverseConversion) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(HistoryReconstructorTest, GetLastConnectivePart) {
|
||||
const testing::MockDataManager data_manager;
|
||||
const dictionary::PosMatcher pos_matcher(data_manager.GetPosMatcherData());
|
||||
const converter::HistoryReconstructor reconstructor(pos_matcher);
|
||||
|
||||
{
|
||||
std::string key;
|
||||
std::string value;
|
||||
uint16_t id = 0;
|
||||
EXPECT_FALSE(reconstructor.GetLastConnectivePart("", &key, &value, &id));
|
||||
EXPECT_FALSE(reconstructor.GetLastConnectivePart(" ", &key, &value, &id));
|
||||
EXPECT_FALSE(reconstructor.GetLastConnectivePart(" ", &key, &value, &id));
|
||||
}
|
||||
|
||||
{
|
||||
std::string key;
|
||||
std::string value;
|
||||
uint16_t id = 0;
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("a", &key, &value, &id));
|
||||
EXPECT_EQ(key, "a");
|
||||
EXPECT_EQ(value, "a");
|
||||
EXPECT_EQ(id, pos_matcher.GetUniqueNounId());
|
||||
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("a ", &key, &value, &id));
|
||||
EXPECT_EQ(key, "a");
|
||||
EXPECT_EQ(value, "a");
|
||||
|
||||
EXPECT_FALSE(reconstructor.GetLastConnectivePart("a ", &key, &value, &id));
|
||||
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("a ", &key, &value, &id));
|
||||
EXPECT_EQ(key, "a");
|
||||
EXPECT_EQ(value, "a");
|
||||
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("a10a", &key, &value, &id));
|
||||
EXPECT_EQ(key, "a");
|
||||
EXPECT_EQ(value, "a");
|
||||
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("a", &key, &value, &id));
|
||||
EXPECT_EQ(key, "a");
|
||||
EXPECT_EQ(value, "a");
|
||||
}
|
||||
|
||||
{
|
||||
std::string key;
|
||||
std::string value;
|
||||
uint16_t id = 0;
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("10", &key, &value, &id));
|
||||
EXPECT_EQ(key, "10");
|
||||
EXPECT_EQ(value, "10");
|
||||
EXPECT_EQ(id, pos_matcher.GetNumberId());
|
||||
|
||||
EXPECT_TRUE(
|
||||
reconstructor.GetLastConnectivePart("10a10", &key, &value, &id));
|
||||
EXPECT_EQ(key, "10");
|
||||
EXPECT_EQ(value, "10");
|
||||
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("10", &key, &value, &id));
|
||||
EXPECT_EQ(key, "10");
|
||||
EXPECT_EQ(value, "10");
|
||||
}
|
||||
|
||||
{
|
||||
std::string key;
|
||||
std::string value;
|
||||
uint16_t id = 0;
|
||||
EXPECT_FALSE(reconstructor.GetLastConnectivePart("あ", &key, &value, &id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ConverterTest, ReconstructHistory) {
|
||||
std::unique_ptr<EngineInterface> engine =
|
||||
MockDataEngineFactory::Create().value();
|
||||
|
161
src/converter/history_reconstructor.cc
Normal file
161
src/converter/history_reconstructor.cc
Normal file
@ -0,0 +1,161 @@
|
||||
// Copyright 2010-2021, Google Inc.
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
// met:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following disclaimer
|
||||
// in the documentation and/or other materials provided with the
|
||||
// distribution.
|
||||
// * Neither the name of Google Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
#include "converter/history_reconstructor.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "base/japanese_util.h"
|
||||
#include "base/util.h"
|
||||
#include "converter/segments.h"
|
||||
#include "dictionary/pos_matcher.h"
|
||||
#include "protocol/commands.pb.h"
|
||||
|
||||
namespace mozc {
|
||||
namespace converter {
|
||||
namespace {
|
||||
// Extracts the last substring that consists of the same script type.
|
||||
// Returns true if the last substring is successfully extracted.
|
||||
// Examples:
|
||||
// - "" -> false
|
||||
// - "x " -> "x" / ALPHABET
|
||||
// - "x " -> false
|
||||
// - "C60" -> "60" / NUMBER
|
||||
// - "200x" -> "x" / ALPHABET
|
||||
// (currently only NUMBER and ALPHABET are supported)
|
||||
bool ExtractLastTokenWithScriptType(const absl::string_view text,
|
||||
std::string *last_token,
|
||||
Util::ScriptType *last_script_type) {
|
||||
last_token->clear();
|
||||
*last_script_type = Util::SCRIPT_TYPE_SIZE;
|
||||
|
||||
ConstChar32ReverseIterator iter(text);
|
||||
if (iter.Done()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Allow one whitespace at the end.
|
||||
if (iter.Get() == ' ') {
|
||||
iter.Next();
|
||||
if (iter.Done()) {
|
||||
return false;
|
||||
}
|
||||
if (iter.Get() == ' ') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<char32_t> reverse_last_token;
|
||||
Util::ScriptType last_script_type_found = Util::GetScriptType(iter.Get());
|
||||
for (; !iter.Done(); iter.Next()) {
|
||||
const char32_t codepoint = iter.Get();
|
||||
if ((codepoint == ' ') ||
|
||||
(Util::GetScriptType(codepoint) != last_script_type_found)) {
|
||||
break;
|
||||
}
|
||||
reverse_last_token.push_back(codepoint);
|
||||
}
|
||||
|
||||
*last_script_type = last_script_type_found;
|
||||
// TODO(yukawa): Replace reverse_iterator with const_reverse_iterator when
|
||||
// build failure on Android is fixed.
|
||||
for (std::vector<char32_t>::reverse_iterator it = reverse_last_token.rbegin();
|
||||
it != reverse_last_token.rend(); ++it) {
|
||||
Util::CodepointToUtf8Append(*it, last_token);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
HistoryReconstructor::HistoryReconstructor(
|
||||
const dictionary::PosMatcher &pos_matcher)
|
||||
: pos_matcher_(pos_matcher) {}
|
||||
|
||||
bool HistoryReconstructor::ReconstructHistory(absl::string_view preceding_text,
|
||||
Segments *segments) const {
|
||||
std::string key;
|
||||
std::string value;
|
||||
uint16_t id;
|
||||
if (!GetLastConnectivePart(preceding_text, &key, &value, &id)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Segment *segment = segments->add_segment();
|
||||
segment->set_key(key);
|
||||
segment->set_segment_type(Segment::HISTORY);
|
||||
Segment::Candidate *candidate = segment->push_back_candidate();
|
||||
candidate->rid = id;
|
||||
candidate->lid = id;
|
||||
candidate->content_key = key;
|
||||
candidate->key = std::move(key);
|
||||
candidate->content_value = value;
|
||||
candidate->value = std::move(value);
|
||||
candidate->attributes = Segment::Candidate::NO_LEARNING;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool HistoryReconstructor::GetLastConnectivePart(
|
||||
const absl::string_view preceding_text, std::string *key,
|
||||
std::string *value, uint16_t *id) const {
|
||||
key->clear();
|
||||
value->clear();
|
||||
*id = pos_matcher_.GetGeneralNounId();
|
||||
|
||||
Util::ScriptType last_script_type = Util::SCRIPT_TYPE_SIZE;
|
||||
std::string last_token;
|
||||
if (!ExtractLastTokenWithScriptType(preceding_text, &last_token,
|
||||
&last_script_type)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Currently only NUMBER and ALPHABET are supported.
|
||||
switch (last_script_type) {
|
||||
case Util::NUMBER: {
|
||||
*key = japanese_util::FullWidthAsciiToHalfWidthAscii(last_token);
|
||||
*value = std::move(last_token);
|
||||
*id = pos_matcher_.GetNumberId();
|
||||
return true;
|
||||
}
|
||||
case Util::ALPHABET: {
|
||||
*key = japanese_util::FullWidthAsciiToHalfWidthAscii(last_token);
|
||||
*value = std::move(last_token);
|
||||
*id = pos_matcher_.GetUniqueNounId();
|
||||
return true;
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace converter
|
||||
} // namespace mozc
|
67
src/converter/history_reconstructor.h
Normal file
67
src/converter/history_reconstructor.h
Normal file
@ -0,0 +1,67 @@
|
||||
// Copyright 2010-2021, Google Inc.
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
// met:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following disclaimer
|
||||
// in the documentation and/or other materials provided with the
|
||||
// distribution.
|
||||
// * Neither the name of Google Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
#ifndef MOZC_CONVERTER_HISTORY_RECONSTRUCTOR_H_
|
||||
#define MOZC_CONVERTER_HISTORY_RECONSTRUCTOR_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include "absl/base/attributes.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "converter/segments.h"
|
||||
#include "dictionary/pos_matcher.h"
|
||||
#include "testing/friend_test.h"
|
||||
|
||||
namespace mozc {
|
||||
namespace converter {
|
||||
|
||||
class HistoryReconstructor {
|
||||
public:
|
||||
explicit HistoryReconstructor(const dictionary::PosMatcher &pos_matcher);
|
||||
|
||||
ABSL_MUST_USE_RESULT
|
||||
bool ReconstructHistory(absl::string_view preceding_text,
|
||||
Segments *segments) const;
|
||||
|
||||
private:
|
||||
FRIEND_TEST(HistoryReconstructorTest, GetLastConnectivePart);
|
||||
|
||||
// Returns the substring of |str|. This substring consists of similar script
|
||||
// type and you can use it as preceding text for conversion.
|
||||
bool GetLastConnectivePart(absl::string_view preceding_text, std::string *key,
|
||||
std::string *value, uint16_t *id) const;
|
||||
|
||||
const dictionary::PosMatcher &pos_matcher_;
|
||||
};
|
||||
|
||||
} // namespace converter
|
||||
} // namespace mozc
|
||||
|
||||
#endif // MOZC_CONVERTER_HISTORY_RECONSTRUCTOR_H_
|
140
src/converter/history_reconstructor_test.cc
Normal file
140
src/converter/history_reconstructor_test.cc
Normal file
@ -0,0 +1,140 @@
|
||||
// Copyright 2010-2021, Google Inc.
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
// met:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following disclaimer
|
||||
// in the documentation and/or other materials provided with the
|
||||
// distribution.
|
||||
// * Neither the name of Google Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
#include "converter/history_reconstructor.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include "converter/segments.h"
|
||||
#include "data_manager/testing/mock_data_manager.h"
|
||||
#include "dictionary/pos_matcher.h"
|
||||
#include "protocol/commands.pb.h"
|
||||
#include "protocol/config.pb.h"
|
||||
#include "protocol/user_dictionary_storage.pb.h"
|
||||
#include "testing/gunit.h"
|
||||
|
||||
namespace mozc {
|
||||
namespace converter {
|
||||
|
||||
TEST(HistoryReconstructorTest, GetLastConnectivePart) {
|
||||
const testing::MockDataManager data_manager;
|
||||
const dictionary::PosMatcher pos_matcher(data_manager.GetPosMatcherData());
|
||||
const converter::HistoryReconstructor reconstructor(pos_matcher);
|
||||
|
||||
{
|
||||
std::string key;
|
||||
std::string value;
|
||||
uint16_t id = 0;
|
||||
EXPECT_FALSE(reconstructor.GetLastConnectivePart("", &key, &value, &id));
|
||||
EXPECT_FALSE(reconstructor.GetLastConnectivePart(" ", &key, &value, &id));
|
||||
EXPECT_FALSE(reconstructor.GetLastConnectivePart(" ", &key, &value, &id));
|
||||
}
|
||||
|
||||
{
|
||||
std::string key;
|
||||
std::string value;
|
||||
uint16_t id = 0;
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("a", &key, &value, &id));
|
||||
EXPECT_EQ(key, "a");
|
||||
EXPECT_EQ(value, "a");
|
||||
EXPECT_EQ(id, pos_matcher.GetUniqueNounId());
|
||||
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("a ", &key, &value, &id));
|
||||
EXPECT_EQ(key, "a");
|
||||
EXPECT_EQ(value, "a");
|
||||
|
||||
EXPECT_FALSE(reconstructor.GetLastConnectivePart("a ", &key, &value, &id));
|
||||
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("a ", &key, &value, &id));
|
||||
EXPECT_EQ(key, "a");
|
||||
EXPECT_EQ(value, "a");
|
||||
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("a10a", &key, &value, &id));
|
||||
EXPECT_EQ(key, "a");
|
||||
EXPECT_EQ(value, "a");
|
||||
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("a", &key, &value, &id));
|
||||
EXPECT_EQ(key, "a");
|
||||
EXPECT_EQ(value, "a");
|
||||
}
|
||||
|
||||
{
|
||||
std::string key;
|
||||
std::string value;
|
||||
uint16_t id = 0;
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("10", &key, &value, &id));
|
||||
EXPECT_EQ(key, "10");
|
||||
EXPECT_EQ(value, "10");
|
||||
EXPECT_EQ(id, pos_matcher.GetNumberId());
|
||||
|
||||
EXPECT_TRUE(
|
||||
reconstructor.GetLastConnectivePart("10a10", &key, &value, &id));
|
||||
EXPECT_EQ(key, "10");
|
||||
EXPECT_EQ(value, "10");
|
||||
|
||||
EXPECT_TRUE(reconstructor.GetLastConnectivePart("10", &key, &value, &id));
|
||||
EXPECT_EQ(key, "10");
|
||||
EXPECT_EQ(value, "10");
|
||||
}
|
||||
|
||||
{
|
||||
std::string key;
|
||||
std::string value;
|
||||
uint16_t id = 0;
|
||||
EXPECT_FALSE(reconstructor.GetLastConnectivePart("あ", &key, &value, &id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(HistoryReconstructorTest, ReconstructHistory) {
|
||||
const testing::MockDataManager data_manager;
|
||||
const dictionary::PosMatcher pos_matcher(data_manager.GetPosMatcherData());
|
||||
const converter::HistoryReconstructor reconstructor(pos_matcher);
|
||||
|
||||
constexpr char kTen[] = "10";
|
||||
|
||||
Segments segments;
|
||||
EXPECT_TRUE(reconstructor.ReconstructHistory(kTen, &segments));
|
||||
EXPECT_EQ(segments.segments_size(), 1);
|
||||
const Segment &segment = segments.segment(0);
|
||||
EXPECT_EQ(segment.segment_type(), Segment::HISTORY);
|
||||
EXPECT_EQ(segment.key(), "10");
|
||||
EXPECT_EQ(segment.candidates_size(), 1);
|
||||
const Segment::Candidate &candidate = segment.candidate(0);
|
||||
EXPECT_EQ(candidate.attributes, Segment::Candidate::NO_LEARNING);
|
||||
EXPECT_EQ(candidate.content_key, "10");
|
||||
EXPECT_EQ(candidate.key, "10");
|
||||
EXPECT_EQ(candidate.content_value, kTen);
|
||||
EXPECT_EQ(candidate.value, kTen);
|
||||
EXPECT_NE(candidate.lid, 0);
|
||||
EXPECT_NE(candidate.rid, 0);
|
||||
}
|
||||
|
||||
} // namespace converter
|
||||
} // namespace mozc
|
Reference in New Issue
Block a user