From 47e88eca0561953091dad3baee578a599d58cd74 Mon Sep 17 00:00:00 2001 From: Hiroyuki Komatsu Date: Mon, 19 Jun 2023 09:07:45 +0000 Subject: [PATCH] Update code for dictionary directories. * Updated LICENSE. * Introduced MOZC_DICT_DIR_COMPONENTS tests for data_manager. * Added mozc_oss_src_dir to directories.gyp. PiperOrigin-RevId: 541564673 --- LICENSE | 83 +++++++++++++++++++ src/base/BUILD.bazel | 4 +- src/converter/converter_main.cc | 3 +- src/converter/converter_test.gyp | 1 + src/converter/pos_id_printer_test.cc | 4 +- src/data/dictionary_oss/BUILD.bazel | 4 +- src/data/test/dictionary/BUILD.bazel | 4 +- src/data_manager/oss/oss_data_manager.gyp | 6 +- .../oss/oss_data_manager_base.gyp | 2 +- src/data_manager/oss/oss_data_manager_test.cc | 7 +- .../oss/oss_data_manager_test.gyp | 38 ++++----- .../testing/mock_data_manager.gyp | 4 +- .../testing/mock_data_manager_base.gyp | 2 +- .../testing/mock_data_manager_test.cc | 6 +- .../testing/mock_data_manager_test.gyp | 10 ++- src/dictionary/BUILD.bazel | 12 ++- src/dictionary/file/codec_test.cc | 4 +- .../system/system_dictionary_test.cc | 2 +- src/gyp/directories.gypi | 2 + src/testing/mozctest.h | 4 + 20 files changed, 155 insertions(+), 47 deletions(-) diff --git a/LICENSE b/LICENSE index c2538be39..15ef0f074 100644 --- a/LICENSE +++ b/LICENSE @@ -26,3 +26,86 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +Files: src/data/dictionary* + +Copyright 2000, 2001, 2002, 2003 Nara Institute of Science +and Technology. All Rights Reserved. + +Use, reproduction, and distribution of this software is permitted. +Any copy of this software, whether in its original form or modified, +must include both the above copyright notice and the following +paragraphs. + +Nara Institute of Science and Technology (NAIST), +the copyright holders, disclaims all warranties with regard to this +software, including all implied warranties of merchantability and +fitness, in no event shall NAIST be liable for +any special, indirect or consequential damages or any damages +whatsoever resulting from loss of use, data or profits, whether in an +action of contract, negligence or other tortuous action, arising out +of or in connection with the use or performance of this software. + +A large portion of the dictionary entries +originate from ICOT Free Software. The following conditions for ICOT +Free Software applies to the current dictionary as well. + +Each User may also freely distribute the Program, whether in its +original form or modified, to any third party or parties, PROVIDED +that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear +on, or be attached to, the Program, which is distributed substantially +in the same form as set out herein and that such intended +distribution, if actually made, will neither violate or otherwise +contravene any of the laws and regulations of the countries having +jurisdiction over the User or the intended distribution itself. + +NO WARRANTY + +The program was produced on an experimental basis in the course of the +research and development conducted during the project and is provided +to users as so produced on an experimental basis. Accordingly, the +program is provided without any warranty whatsoever, whether express, +implied, statutory or otherwise. The term "warranty" used herein +includes, but is not limited to, any warranty of the quality, +performance, merchantability and fitness for a particular purpose of +the program and the nonexistence of any infringement or violation of +any right of any third party. + +Each user of the program will agree and understand, and be deemed to +have agreed and understood, that there is no warranty whatsoever for +the program and, accordingly, the entire risk arising from or +otherwise connected with the program is assumed by the user. + +Therefore, neither ICOT, the copyright holder, or any other +organization that participated in or was otherwise related to the +development of the program and their respective officials, directors, +officers and other employees shall be held liable for any and all +damages, including, without limitation, general, special, incidental +and consequential damages, arising out of or otherwise in connection +with the use or inability to use the program or any product, material +or result produced or otherwise obtained by using the program, +regardless of whether they have been advised of, or otherwise had +knowledge of, the possibility of such damages at any time during the +project or thereafter. Each user will be deemed to have agreed to the +foregoing by his or her commencement of use of the program. The term +"use" as used herein includes, but is not limited to, the use, +modification, copying and distribution of the program and the +production of secondary products from the program. + +In the case where the program, whether in its original form or +modified, was distributed or delivered to or received by a user from +any person, organization or entity other than ICOT, unless it makes or +grants independently of ICOT any specific warranty to the user in +writing, such person, organization or entity, will also be exempted +from and not be held liable to the user for any such damages as noted +above as far as the program is concerned. + +--- + +Files: src/data/dictionary* + +Public Domain Dataです。使用・変更・配布に関しては一切の制限をつけません。 +商品などに組み込むことも自由に行なってください。すでにいくつかの辞書には沖縄辞書が採用されています。 +勝手ながら、沖縄辞書に寄贈された辞書も in the Public Domain' 扱いとさせていただきます。 diff --git a/src/base/BUILD.bazel b/src/base/BUILD.bazel index 226df725b..a3dbd54ff 100644 --- a/src/base/BUILD.bazel +++ b/src/base/BUILD.bazel @@ -823,7 +823,9 @@ genrule( " --target_platform=" + TARGET_PLATFORM ), tools = ["//build_tools:mozc_version"], - visibility = ["//:__subpackages__"], + visibility = [ + "//:__subpackages__", + ], ) genrule( diff --git a/src/converter/converter_main.cc b/src/converter/converter_main.cc index 621c5fd2d..988768985 100644 --- a/src/converter/converter_main.cc +++ b/src/converter/converter_main.cc @@ -460,8 +460,9 @@ int main(int argc, char **argv) { << "--engine_name=" << absl::GetFlag(FLAGS_engine_name); if (absl::GetFlag(FLAGS_id_def).empty()) { + std::string mozc_idfile_dir = "."; absl::SetFlag(&FLAGS_id_def, - mozc::SelectIdDefFromName(mozc_runfiles_dir, + mozc::SelectIdDefFromName(mozc_idfile_dir, absl::GetFlag(FLAGS_engine_name))); } diff --git a/src/converter/converter_test.gyp b/src/converter/converter_test.gyp index 455bcf00b..e2fd4b2ad 100644 --- a/src/converter/converter_test.gyp +++ b/src/converter/converter_test.gyp @@ -53,6 +53,7 @@ '../config/config.gyp:config_handler', '../data_manager/testing/mock_data_manager.gyp:mock_data_manager', '../dictionary/dictionary.gyp:suffix_dictionary', + '../dictionary/dictionary_base.gyp:suppression_dictionary', '../dictionary/dictionary_base.gyp:user_dictionary', '../dictionary/dictionary_base.gyp:user_pos', '../dictionary/system/system_dictionary.gyp:system_dictionary', diff --git a/src/converter/pos_id_printer_test.cc b/src/converter/pos_id_printer_test.cc index b1bec36f7..c0747cfe3 100644 --- a/src/converter/pos_id_printer_test.cc +++ b/src/converter/pos_id_printer_test.cc @@ -37,8 +37,8 @@ namespace mozc { namespace internal { TEST(PosIdPrinterTest, BasicIdTest) { - PosIdPrinter printer(InputFileStream( - testing::GetSourceFileOrDie({"data", "test", "dictionary", "id.def"}))); + PosIdPrinter printer(InputFileStream(testing::GetSourceFileOrDie( + {MOZC_DICT_DIR_COMPONENTS, "test", "dictionary", "id.def"}))); EXPECT_EQ(printer.IdToString(1934), "名詞,サ変接続,*,*,*,*,*"); EXPECT_EQ(printer.IdToString(1935), "名詞,サ変接続,*,*,*,*,*,使用"); diff --git a/src/data/dictionary_oss/BUILD.bazel b/src/data/dictionary_oss/BUILD.bazel index 267dacb27..5fc9a97d3 100644 --- a/src/data/dictionary_oss/BUILD.bazel +++ b/src/data/dictionary_oss/BUILD.bazel @@ -29,7 +29,9 @@ load("//converter:evaluation.bzl", "evaluation") -package(default_visibility = ["//:__subpackages__"]) +package(default_visibility = [ + "//:__subpackages__", +]) exports_files([ "collocation.txt", diff --git a/src/data/test/dictionary/BUILD.bazel b/src/data/test/dictionary/BUILD.bazel index 32c74083c..384b217fa 100644 --- a/src/data/test/dictionary/BUILD.bazel +++ b/src/data/test/dictionary/BUILD.bazel @@ -27,7 +27,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -package(default_visibility = ["//:__subpackages__"]) +package(default_visibility = [ + "//:__subpackages__", +]) exports_files([ "collocation.txt", diff --git a/src/data_manager/oss/oss_data_manager.gyp b/src/data_manager/oss/oss_data_manager.gyp index bc9f96db2..67ce27b16 100644 --- a/src/data_manager/oss/oss_data_manager.gyp +++ b/src/data_manager/oss/oss_data_manager.gyp @@ -37,7 +37,7 @@ 'current_dir': '.', 'mozc_dir': '../..', 'common_data_dir': '<(mozc_dir)/data', - 'platform_data_dir': '<(mozc_dir)/data/dictionary_oss', + 'platform_data_dir': '<(mozc_oss_src_dir)/data/dictionary_oss', 'boundary_def': '<(mozc_dir)/data/rules/boundary.def', 'dataset_tag': 'oss', 'use_1byte_cost_for_connection_data': 'false', @@ -53,8 +53,8 @@ '<(platform_data_dir)/dictionary08.txt', '<(platform_data_dir)/dictionary09.txt', '<(platform_data_dir)/reading_correction.tsv', - '<(mozc_dir)/data/dictionary_manual/domain.txt', - '<(mozc_dir)/data/dictionary_manual/era.txt', + '<(mozc_oss_src_dir)/data/dictionary_manual/domain.txt', + '<(mozc_oss_src_dir)/data/dictionary_manual/era.txt', ], # Hex-escaped string of "\xEFMOZC\r\n" 'magic_number': "\\xEF\\x4D\\x4F\\x5A\\x43\\x0D\\x0A", diff --git a/src/data_manager/oss/oss_data_manager_base.gyp b/src/data_manager/oss/oss_data_manager_base.gyp index 769a47909..92ab20a54 100644 --- a/src/data_manager/oss/oss_data_manager_base.gyp +++ b/src/data_manager/oss/oss_data_manager_base.gyp @@ -37,7 +37,7 @@ 'current_dir': '.', 'mozc_dir': '../..', 'common_data_dir': '<(mozc_dir)/data', - 'platform_data_dir': '<(mozc_dir)/data/dictionary_oss', + 'platform_data_dir': '<(mozc_oss_src_dir)/data/dictionary_oss', 'dataset_tag': 'oss', }, # This 'includes' defines the following targets: diff --git a/src/data_manager/oss/oss_data_manager_test.cc b/src/data_manager/oss/oss_data_manager_test.cc index 98f4f592e..0f39732ce 100644 --- a/src/data_manager/oss/oss_data_manager_test.cc +++ b/src/data_manager/oss/oss_data_manager_test.cc @@ -60,13 +60,14 @@ class OssDataManagerTest : public DataManagerTestBase { {"data_manager", "oss", "connection_single_column.txt"}), 1, mozc::testing::GetSourceFilesInDirOrDie( - {"data", "dictionary_oss"}, + {MOZC_DICT_DIR_COMPONENTS, "dictionary_oss"}, {"dictionary00.txt", "dictionary01.txt", "dictionary02.txt", "dictionary03.txt", "dictionary04.txt", "dictionary05.txt", "dictionary06.txt", "dictionary07.txt", "dictionary08.txt", "dictionary09.txt"}), - mozc::testing::GetSourceFilesInDirOrDie({"data", "dictionary_oss"}, - {"suggestion_filter.txt"}), + mozc::testing::GetSourceFilesInDirOrDie( + {MOZC_DICT_DIR_COMPONENTS, "dictionary_oss"}, + {"suggestion_filter.txt"}), { GetTypingModelEntry("typing_model_12keys-hiragana.tsv"), GetTypingModelEntry("typing_model_flick-hiragana.tsv"), diff --git a/src/data_manager/oss/oss_data_manager_test.gyp b/src/data_manager/oss/oss_data_manager_test.gyp index 224027fe9..fd6d8b6e2 100644 --- a/src/data_manager/oss/oss_data_manager_test.gyp +++ b/src/data_manager/oss/oss_data_manager_test.gyp @@ -56,27 +56,25 @@ { 'target_name': 'install_oss_data_manager_test_data', 'type': 'none', - 'variables': { - 'test_data': [ - '<(gen_out_dir)/connection_single_column.txt', - '../../<(test_data_subdir)/dictionary00.txt', - '../../<(test_data_subdir)/dictionary01.txt', - '../../<(test_data_subdir)/dictionary02.txt', - '../../<(test_data_subdir)/dictionary03.txt', - '../../<(test_data_subdir)/dictionary04.txt', - '../../<(test_data_subdir)/dictionary05.txt', - '../../<(test_data_subdir)/dictionary06.txt', - '../../<(test_data_subdir)/dictionary07.txt', - '../../<(test_data_subdir)/dictionary08.txt', - '../../<(test_data_subdir)/dictionary09.txt', - '../../<(test_data_subdir)/suggestion_filter.txt' - ], - 'test_data_subdir': 'data/dictionary_oss', - }, - 'dependencies': [ - 'oss_data_manager.gyp:gen_connection_single_column_txt_for_oss#host', + 'copies': [ + { + 'destination': '<(SHARED_INTERMEDIATE_DIR)/third_party/mozc/src/data/dictionary_oss/', + 'files': [ + '<(gen_out_dir)/connection_single_column.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/dictionary00.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/dictionary01.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/dictionary02.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/dictionary03.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/dictionary04.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/dictionary05.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/dictionary06.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/dictionary07.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/dictionary08.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/dictionary09.txt', + '<(mozc_oss_src_dir)/data/dictionary_oss/suggestion_filter.txt', + ], + }, ], - 'includes': ['../../gyp/install_testdata.gypi'], }, ], } diff --git a/src/data_manager/testing/mock_data_manager.gyp b/src/data_manager/testing/mock_data_manager.gyp index dd9d3b5f7..876648d3f 100644 --- a/src/data_manager/testing/mock_data_manager.gyp +++ b/src/data_manager/testing/mock_data_manager.gyp @@ -37,13 +37,13 @@ 'current_dir': '.', 'mozc_dir': '../..', 'common_data_dir': '<(mozc_dir)/data', - 'platform_data_dir': '<(mozc_dir)/data/test/dictionary', + 'platform_data_dir': '<(mozc_oss_src_dir)/data/test/dictionary', 'boundary_def': '<(mozc_dir)/data/rules/boundary.def', 'dataset_tag': 'mock', 'use_1byte_cost_for_connection_data': 'false', 'dictionary_files': [ '<(platform_data_dir)/dictionary.txt', - '<(mozc_dir)/data/dictionary_manual/domain.txt', + '<(mozc_oss_src_dir)/data/dictionary_manual/domain.txt', ], 'magic_number': '\\x4D\\x4F\\x43\\x4B', # MOCK 'mozc_data_varname': 'kMockMozcDataSet', diff --git a/src/data_manager/testing/mock_data_manager_base.gyp b/src/data_manager/testing/mock_data_manager_base.gyp index 26c9fc97b..21c75acbf 100644 --- a/src/data_manager/testing/mock_data_manager_base.gyp +++ b/src/data_manager/testing/mock_data_manager_base.gyp @@ -37,7 +37,7 @@ 'current_dir': '.', 'mozc_dir': '../..', 'common_data_dir': '<(mozc_dir)/data', - 'platform_data_dir': '<(mozc_dir)/data/test/dictionary', + 'platform_data_dir': '<(mozc_oss_src_dir)/data/test/dictionary', 'dataset_tag': 'mock', }, # This 'includes' defines the following targets: diff --git a/src/data_manager/testing/mock_data_manager_test.cc b/src/data_manager/testing/mock_data_manager_test.cc index 39d6ce719..93a64bb6a 100644 --- a/src/data_manager/testing/mock_data_manager_test.cc +++ b/src/data_manager/testing/mock_data_manager_test.cc @@ -60,9 +60,11 @@ class MockDataManagerTest : public DataManagerTestBase { {"data_manager", "testing", "connection_single_column.txt"}), 1, mozc::testing::GetSourceFilesInDirOrDie( - {"data", "test", "dictionary"}, {"dictionary.txt"}), + {MOZC_DICT_DIR_COMPONENTS, "test", "dictionary"}, + {"dictionary.txt"}), mozc::testing::GetSourceFilesInDirOrDie( - {"data", "test", "dictionary"}, {"suggestion_filter.txt"}), + {MOZC_DICT_DIR_COMPONENTS, "test", "dictionary"}, + {"suggestion_filter.txt"}), { GetTypingModelEntry("typing_model_12keys-hiragana.tsv"), GetTypingModelEntry("typing_model_flick-hiragana.tsv"), diff --git a/src/data_manager/testing/mock_data_manager_test.gyp b/src/data_manager/testing/mock_data_manager_test.gyp index b3f524d1e..6970ca6ea 100644 --- a/src/data_manager/testing/mock_data_manager_test.gyp +++ b/src/data_manager/testing/mock_data_manager_test.gyp @@ -54,9 +54,11 @@ }, 'copies': [ { - 'destination': '<(mozc_data_dir)/data/test/dictionary/', - 'files': [ '../../data/test/dictionary/dictionary.txt', - '../../data/test/dictionary/suggestion_filter.txt', ], + 'destination': '<(SHARED_INTERMEDIATE_DIR)/third_party/mozc/src/data/test/dictionary/', + 'files': [ + '<(mozc_oss_src_dir)/data/test/dictionary/dictionary.txt', + '<(mozc_oss_src_dir)/data/test/dictionary/suggestion_filter.txt', + ], }, ], }, @@ -68,7 +70,7 @@ ], 'copies': [ { - 'destination': '<(mozc_data_dir)/data/test/dictionary/', + 'destination': '<(SHARED_INTERMEDIATE_DIR)/third_party/mozc/src/data/test/dictionary/', 'files': [ '<(gen_out_dir)/connection_single_column.txt', ], diff --git a/src/dictionary/BUILD.bazel b/src/dictionary/BUILD.bazel index d660e3408..aa86fcbc1 100644 --- a/src/dictionary/BUILD.bazel +++ b/src/dictionary/BUILD.bazel @@ -224,13 +224,17 @@ mozc_cc_library( mozc_py_binary( name = "gen_aux_dictionary", srcs = ["gen_aux_dictionary.py"], - visibility = ["//:__subpackages__"], + visibility = [ + "//:__subpackages__", + ], ) mozc_py_binary( name = "gen_filtered_dictionary", srcs = ["gen_filtered_dictionary.py"], - visibility = ["//:__subpackages__"], + visibility = [ + "//:__subpackages__", + ], ) mozc_cc_library( @@ -756,5 +760,7 @@ run_binary( ], ), tool = ":gen_zip_code_seed", - visibility = ["//:__subpackages__"], + visibility = [ + "//:__subpackages__", + ], ) diff --git a/src/dictionary/file/codec_test.cc b/src/dictionary/file/codec_test.cc index 3ad6ccbc7..a2d1c8341 100644 --- a/src/dictionary/file/codec_test.cc +++ b/src/dictionary/file/codec_test.cc @@ -55,7 +55,9 @@ namespace { class CodecTest : public ::testing::Test { public: - CodecTest() : test_file_(absl::GetFlag(FLAGS_test_tmpdir) + "testfile.txt") {} + CodecTest() + : test_file_(FileUtil::JoinPath(absl::GetFlag(FLAGS_test_tmpdir), + "testfile.txt")) {} protected: void SetUp() override { diff --git a/src/dictionary/system/system_dictionary_test.cc b/src/dictionary/system/system_dictionary_test.cc index b4c4a5ead..4ee8ac8c3 100644 --- a/src/dictionary/system/system_dictionary_test.cc +++ b/src/dictionary/system/system_dictionary_test.cc @@ -79,7 +79,7 @@ class SystemDictionaryTest : public ::testing::Test { dic_fn_( FileUtil::JoinPath(absl::GetFlag(FLAGS_test_tmpdir), "mozc.dic")) { const std::string dic_path = mozc::testing::GetSourceFileOrDie( - {"data", "dictionary_oss", "dictionary00.txt"}); + {MOZC_DICT_DIR_COMPONENTS, "dictionary_oss", "dictionary00.txt"}); text_dict_.LoadWithLineLimit(dic_path, "", absl::GetFlag(FLAGS_dictionary_test_size)); diff --git a/src/gyp/directories.gypi b/src/gyp/directories.gypi index 3c5df3775..09961b4d0 100644 --- a/src/gyp/directories.gypi +++ b/src/gyp/directories.gypi @@ -35,6 +35,8 @@ 'absl_srcdir': '<(absl_dir)/absl', 'absl_include_dirs': ['<(absl_dir)'], + 'mozc_oss_src_dir': '<(DEPTH)', + # TODO(komatsu): This can be replaced with 'android_ndk_dir'. 'mozc_build_tools_dir': '<(abs_depth)/<(build_short_base)/mozc_build_tools', diff --git a/src/testing/mozctest.h b/src/testing/mozctest.h index 2be90063f..3c7eb9406 100644 --- a/src/testing/mozctest.h +++ b/src/testing/mozctest.h @@ -95,6 +95,10 @@ class ScopedTempUserProfileDirectory { const std::string original_dir_; }; +#ifndef MOZC_DICT_DIR_COMPONENTS +#define MOZC_DICT_DIR_COMPONENTS "data" +#endif // MOZC_DICT_DIR_COMPONENTS + } // namespace testing } // namespace mozc