From 1df1e236440e32fc12560e7b9f6e4f289fa80f50 Mon Sep 17 00:00:00 2001 From: GunChleoc Date: Sun, 10 Mar 2024 11:52:56 +0000 Subject: [PATCH 1/2] Unified array for SUPPORTED_LOCALES and add handling for all locale codes to ActivityPub::CaseTransform --- app/helpers/languages_helper.rb | 72 ++++++++++++--------------- app/lib/activitypub/case_transform.rb | 3 +- lib/tasks/repo.rake | 2 +- spec/helpers/languages_helper_spec.rb | 2 +- 4 files changed, 37 insertions(+), 42 deletions(-) diff --git a/app/helpers/languages_helper.rb b/app/helpers/languages_helper.rb index 9e1c0a7db1d..4427fee8d17 100644 --- a/app/helpers/languages_helper.rb +++ b/app/helpers/languages_helper.rb @@ -1,7 +1,10 @@ # frozen_string_literal: true module LanguagesHelper - ISO_639_1 = { + # These locales can be selected as posting languages and filtered for by users. + # When adding a locale, prefer using a ISO-639-1 (2-letter) locale over + # using a ISO-639-3 (3-letter) locale if available for the language. + SUPPORTED_LOCALES = { aa: ['Afar', 'Afaraf'].freeze, ab: ['Abkhaz', 'аҧсуа бызшәа'].freeze, ae: ['Avestan', 'avesta'].freeze, @@ -11,6 +14,7 @@ module LanguagesHelper an: ['Aragonese', 'aragonés'].freeze, ar: ['Arabic', 'اللغة العربية'].freeze, as: ['Assamese', 'অসমীয়া'].freeze, + ast: ['Asturian', 'Asturianu'].freeze, av: ['Avaric', 'авар мацӀ'].freeze, ay: ['Aymara', 'aymar aru'].freeze, az: ['Azerbaijani', 'azərbaycan dili'].freeze, @@ -27,9 +31,13 @@ module LanguagesHelper ca: ['Catalan', 'Català'].freeze, ce: ['Chechen', 'нохчийн мотт'].freeze, ch: ['Chamorro', 'Chamoru'].freeze, + chr: ['Cherokee', 'ᏣᎳᎩ ᎦᏬᏂᎯᏍᏗ'].freeze, + ckb: ['Sorani (Kurdish)', 'سۆرانی'].freeze, co: ['Corsican', 'corsu'].freeze, + cnr: ['Montenegrin', 'crnogorski'].freeze, cr: ['Cree', 'ᓀᐦᐃᔭᐍᐏᐣ'].freeze, cs: ['Czech', 'čeština'].freeze, + csb: ['Kashubian', 'Kaszëbsczi'].freeze, cu: ['Old Church Slavonic', 'ѩзыкъ словѣньскъ'].freeze, cv: ['Chuvash', 'чӑваш чӗлхи'].freeze, cy: ['Welsh', 'Cymraeg'].freeze, @@ -76,8 +84,10 @@ module LanguagesHelper it: ['Italian', 'Italiano'].freeze, iu: ['Inuktitut', 'ᐃᓄᒃᑎᑐᑦ'].freeze, ja: ['Japanese', '日本語'].freeze, + jbo: ['Lojban', 'la .lojban.'].freeze, jv: ['Javanese', 'basa Jawa'].freeze, ka: ['Georgian', 'ქართული'].freeze, + kab: ['Kabyle', 'Taqbaylit'].freeze, kg: ['Kongo', 'Kikongo'].freeze, ki: ['Kikuyu', 'Gĩkũyũ'].freeze, kj: ['Kwanyama', 'Kuanyama'].freeze, @@ -94,6 +104,8 @@ module LanguagesHelper ky: ['Kyrgyz', 'Кыргызча'].freeze, la: ['Latin', 'latine'].freeze, lb: ['Luxembourgish', 'Lëtzebuergesch'].freeze, + ldn: ['Láadan', 'Láadan'].freeze, + lfn: ['Lingua Franca Nova', 'lingua franca nova'].freeze, lg: ['Ganda', 'Luganda'].freeze, li: ['Limburgish', 'Limburgs'].freeze, ln: ['Lingala', 'Lingála'].freeze, @@ -107,6 +119,7 @@ module LanguagesHelper mk: ['Macedonian', 'македонски јазик'].freeze, ml: ['Malayalam', 'മലയാളം'].freeze, mn: ['Mongolian', 'Монгол хэл'].freeze, + moh: ['Mohawk', 'Kanienʼkéha'].freeze, mr: ['Marathi', 'मराठी'].freeze, ms: ['Malay', 'Bahasa Melayu'].freeze, 'ms-Arab': ['Jawi Malay', 'بهاس ملايو'].freeze, @@ -115,6 +128,7 @@ module LanguagesHelper na: ['Nauru', 'Ekakairũ Naoero'].freeze, nb: ['Norwegian Bokmål', 'Norsk bokmål'].freeze, nd: ['Northern Ndebele', 'isiNdebele'].freeze, + nds: ['Low German', 'Plattdüütsch'].freeze, ne: ['Nepali', 'नेपाली'].freeze, ng: ['Ndonga', 'Owambo'].freeze, nl: ['Dutch', 'Nederlands'].freeze, @@ -129,6 +143,7 @@ module LanguagesHelper or: ['Oriya', 'ଓଡ଼ିଆ'].freeze, os: ['Ossetian', 'ирон æвзаг'].freeze, pa: ['Punjabi', 'ਪੰਜਾਬੀ'].freeze, + pdc: ['Pennsylvania Dutch', 'Pennsilfaani-Deitsch'].freeze, pi: ['Pāli', 'पाऴि'].freeze, pl: ['Polish', 'Polski'].freeze, ps: ['Pashto', 'پښتو'].freeze, @@ -141,12 +156,15 @@ module LanguagesHelper rw: ['Kinyarwanda', 'Ikinyarwanda'].freeze, sa: ['Sanskrit', 'संस्कृतम्'].freeze, sc: ['Sardinian', 'sardu'].freeze, + sco: ['Scots', 'Scots'].freeze, sd: ['Sindhi', 'सिन्धी'].freeze, se: ['Northern Sami', 'Davvisámegiella'].freeze, sg: ['Sango', 'yângâ tî sängö'].freeze, si: ['Sinhala', 'සිංහල'].freeze, sk: ['Slovak', 'slovenčina'].freeze, sl: ['Slovenian', 'slovenščina'].freeze, + sma: ['Southern Sami', 'Åarjelsaemien Gïele'].freeze, + smj: ['Lule Sami', 'Julevsámegiella'].freeze, sn: ['Shona', 'chiShona'].freeze, so: ['Somali', 'Soomaaliga'].freeze, sq: ['Albanian', 'Shqip'].freeze, @@ -156,6 +174,7 @@ module LanguagesHelper su: ['Sundanese', 'Basa Sunda'].freeze, sv: ['Swedish', 'Svenska'].freeze, sw: ['Swahili', 'Kiswahili'].freeze, + szl: ['Silesian', 'ślůnsko godka'].freeze, ta: ['Tamil', 'தமிழ்'].freeze, te: ['Telugu', 'తెలుగు'].freeze, tg: ['Tajik', 'тоҷикӣ'].freeze, @@ -165,6 +184,7 @@ module LanguagesHelper tl: ['Tagalog', 'Wikang Tagalog'].freeze, tn: ['Tswana', 'Setswana'].freeze, to: ['Tonga', 'faka Tonga'].freeze, + tok: ['Toki Pona', 'toki pona'].freeze, tr: ['Turkish', 'Türkçe'].freeze, ts: ['Tsonga', 'Xitsonga'].freeze, tt: ['Tatar', 'татар теле'].freeze, @@ -174,59 +194,33 @@ module LanguagesHelper uk: ['Ukrainian', 'Українська'].freeze, ur: ['Urdu', 'اردو'].freeze, uz: ['Uzbek', 'Ўзбек'].freeze, + vai: ['Vai', 'ꕙꔤ'].freeze, ve: ['Venda', 'Tshivenḓa'].freeze, vi: ['Vietnamese', 'Tiếng Việt'].freeze, vo: ['Volapük', 'Volapük'].freeze, wa: ['Walloon', 'walon'].freeze, wo: ['Wolof', 'Wollof'].freeze, + xal: ['Kalmyk', 'Хальмг келн'].freeze, xh: ['Xhosa', 'isiXhosa'].freeze, yi: ['Yiddish', 'ייִדיש'].freeze, yo: ['Yoruba', 'Yorùbá'].freeze, za: ['Zhuang', 'Saɯ cueŋƅ'].freeze, - zh: ['Chinese', '中文'].freeze, - zu: ['Zulu', 'isiZulu'].freeze, - }.freeze - - ISO_639_3 = { - ast: ['Asturian', 'Asturianu'].freeze, - chr: ['Cherokee', 'ᏣᎳᎩ ᎦᏬᏂᎯᏍᏗ'].freeze, - ckb: ['Sorani (Kurdish)', 'سۆرانی'].freeze, - cnr: ['Montenegrin', 'crnogorski'].freeze, - csb: ['Kashubian', 'Kaszëbsczi'].freeze, - jbo: ['Lojban', 'la .lojban.'].freeze, - kab: ['Kabyle', 'Taqbaylit'].freeze, - ldn: ['Láadan', 'Láadan'].freeze, - lfn: ['Lingua Franca Nova', 'lingua franca nova'].freeze, - moh: ['Mohawk', 'Kanienʼkéha'].freeze, - nds: ['Low German', 'Plattdüütsch'].freeze, - pdc: ['Pennsylvania Dutch', 'Pennsilfaani-Deitsch'].freeze, - sco: ['Scots', 'Scots'].freeze, - sma: ['Southern Sami', 'Åarjelsaemien Gïele'].freeze, - smj: ['Lule Sami', 'Julevsámegiella'].freeze, - szl: ['Silesian', 'ślůnsko godka'].freeze, - tok: ['Toki Pona', 'toki pona'].freeze, - vai: ['Vai', 'ꕙꔤ'].freeze, - xal: ['Kalmyk', 'Хальмг келн'].freeze, zba: ['Balaibalan', 'باليبلن'].freeze, zgh: ['Standard Moroccan Tamazight', 'ⵜⴰⵎⴰⵣⵉⵖⵜ'].freeze, - }.freeze - - # e.g. For Chinese, which is not a language, - # but a language family in spite of sharing the main locale code - # We need to be able to filter these - ISO_639_1_REGIONAL = { + zh: ['Chinese', '中文'].freeze, + # "Chinese" is not a language, but a language family in spite of + # sharing the main locale code. 'zh-CN': ['Chinese (China)', '简体中文'].freeze, 'zh-HK': ['Chinese (Hong Kong)', '繁體中文（香港）'].freeze, 'zh-TW': ['Chinese (Taiwan)', '繁體中文（臺灣）'].freeze, 'zh-YUE': ['Cantonese', '廣東話'].freeze, + zu: ['Zulu', 'isiZulu'].freeze, }.freeze - SUPPORTED_LOCALES = {}.merge(ISO_639_1).merge(ISO_639_1_REGIONAL).merge(ISO_639_3).freeze - - # For ISO-639-1 and ISO-639-3 language codes, we have their official - # names, but for some translations, we need the names of the - # regional variants specifically - REGIONAL_LOCALE_NAMES = { + # These locales are being translated on Crowdin and are available in the UI, + # but for presenting posting and filtering languages, we drop the country code and + # fall back to the language code, thus skipping these locales there. + UI_ONLY_REGIONAL_LOCALES = { 'en-GB': 'English (British)', 'es-AR': 'Español (Argentina)', 'es-MX': 'Español (México)', @@ -242,7 +236,7 @@ module LanguagesHelper '000' elsif (supported_locale = SUPPORTED_LOCALES[locale.to_sym]) ASCIIFolding.new.fold(supported_locale[1]).downcase - elsif (regional_locale = REGIONAL_LOCALE_NAMES[locale.to_sym]) + elsif (regional_locale = UI_ONLY_REGIONAL_LOCALES[locale.to_sym]) ASCIIFolding.new.fold(regional_locale).downcase else locale @@ -259,7 +253,7 @@ module LanguagesHelper I18n.t('generic.none') elsif (supported_locale = SUPPORTED_LOCALES[locale.to_sym]) supported_locale[1] - elsif (regional_locale = REGIONAL_LOCALE_NAMES[locale.to_sym]) + elsif (regional_locale = UI_ONLY_REGIONAL_LOCALES[locale.to_sym]) regional_locale else locale diff --git a/app/lib/activitypub/case_transform.rb b/app/lib/activitypub/case_transform.rb index bf5de722103..93b4b988879 100644 --- a/app/lib/activitypub/case_transform.rb +++ b/app/lib/activitypub/case_transform.rb @@ -14,7 +14,8 @@ module ActivityPub::CaseTransform when String camel_lower_cache[value] ||= if value.start_with?('_:') "_:#{value.delete_prefix('_:').underscore.camelize(:lower)}" - elsif LanguagesHelper::ISO_639_1_REGIONAL.key?(value.to_sym) + elsif LanguagesHelper::SUPPORTED_LOCALES.key?(value.to_sym) || + LanguagesHelper::UI_ONLY_REGIONAL_LOCALES.key?(value.to_sym) value else value.underscore.camelize(:lower) diff --git a/lib/tasks/repo.rake b/lib/tasks/repo.rake index 1ed1ee5c323..e5dfb570d4f 100644 --- a/lib/tasks/repo.rake +++ b/lib/tasks/repo.rake @@ -96,7 +96,7 @@ namespace :repo do end.uniq.compact missing_available_locales = locales_in_files - I18n.available_locales - supported_locale_codes = Set.new(LanguagesHelper::SUPPORTED_LOCALES.keys + LanguagesHelper::REGIONAL_LOCALE_NAMES.keys) + supported_locale_codes = Set.new(LanguagesHelper::SUPPORTED_LOCALES.keys + LanguagesHelper::UI_ONLY_REGIONAL_LOCALES.keys) missing_locale_names = I18n.available_locales.reject { |locale| supported_locale_codes.include?(locale) } critical = false diff --git a/spec/helpers/languages_helper_spec.rb b/spec/helpers/languages_helper_spec.rb index 99461b293ba..eae54b61a46 100644 --- a/spec/helpers/languages_helper_spec.rb +++ b/spec/helpers/languages_helper_spec.rb @@ -5,7 +5,7 @@ require 'rails_helper' describe LanguagesHelper do describe 'the SUPPORTED_LOCALES constant' do it 'includes all i18n locales' do - expect(Set.new(described_class::SUPPORTED_LOCALES.keys + described_class::REGIONAL_LOCALE_NAMES.keys)).to include(*I18n.available_locales) + expect(Set.new(described_class::SUPPORTED_LOCALES.keys + described_class::UI_ONLY_REGIONAL_LOCALES.keys)).to include(*I18n.available_locales) end end From a1414f00416f6000c291786b013be8c723c78569 Mon Sep 17 00:00:00 2001 From: GunChleoc Date: Sun, 10 Mar 2024 12:22:24 +0000 Subject: [PATCH 2/2] Unify array structure for locales --- app/helpers/languages_helper.rb | 32 +++++++++++++-------------- app/lib/activitypub/case_transform.rb | 3 +-- lib/tasks/repo.rake | 2 +- spec/helpers/languages_helper_spec.rb | 2 +- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/app/helpers/languages_helper.rb b/app/helpers/languages_helper.rb index 4427fee8d17..2a7677033ce 100644 --- a/app/helpers/languages_helper.rb +++ b/app/helpers/languages_helper.rb @@ -221,23 +221,23 @@ module LanguagesHelper # but for presenting posting and filtering languages, we drop the country code and # fall back to the language code, thus skipping these locales there. UI_ONLY_REGIONAL_LOCALES = { - 'en-GB': 'English (British)', - 'es-AR': 'Español (Argentina)', - 'es-MX': 'Español (México)', - 'fr-CA': 'Français (Canadien)', - 'pt-BR': 'Português (Brasil)', - 'pt-PT': 'Português (Portugal)', - 'sr-Latn': 'Srpski (latinica)', + 'en-GB': ['English (British)', 'English (British)'].freeze, + 'es-AR': ['Spanish (Argentina)', 'Español (Argentina)'].freeze, + 'es-MX': ['Spanish (Mexico)', 'Español (México)'].freeze, + 'fr-CA': ['French (Canadian)', 'Français (Canadien)'].freeze, + 'pt-BR': ['Portuguese (Brasil)', 'Português (Brasil)'].freeze, + 'pt-PT': ['Portuguese (Portugal)', 'Português (Portugal)'].freeze, + 'sr-Latn': ['Serbian (Latin)', 'Srpski (latinica)'].freeze, }.freeze + KNOWN_LOCALES = {}.merge(SUPPORTED_LOCALES).merge(UI_ONLY_REGIONAL_LOCALES).freeze + # Helper for self.sorted_locale_keys private_class_method def self.locale_name_for_sorting(locale) if locale.blank? || locale == 'und' '000' - elsif (supported_locale = SUPPORTED_LOCALES[locale.to_sym]) - ASCIIFolding.new.fold(supported_locale[1]).downcase - elsif (regional_locale = UI_ONLY_REGIONAL_LOCALES[locale.to_sym]) - ASCIIFolding.new.fold(regional_locale).downcase + elsif (known_locale = KNOWN_LOCALES[locale.to_sym]) + ASCIIFolding.new.fold(known_locale[1]).downcase else locale end @@ -251,10 +251,8 @@ module LanguagesHelper def native_locale_name(locale) if locale.blank? || locale == 'und' I18n.t('generic.none') - elsif (supported_locale = SUPPORTED_LOCALES[locale.to_sym]) - supported_locale[1] - elsif (regional_locale = UI_ONLY_REGIONAL_LOCALES[locale.to_sym]) - regional_locale + elsif (known_locale = KNOWN_LOCALES[locale.to_sym]) + known_locale[1] else locale end @@ -263,8 +261,8 @@ module LanguagesHelper def standard_locale_name(locale) if locale.blank? I18n.t('generic.none') - elsif (supported_locale = SUPPORTED_LOCALES[locale.to_sym]) - supported_locale[0] + elsif (known_locale = KNOWN_LOCALES[locale.to_sym]) + known_locale[0] else locale end diff --git a/app/lib/activitypub/case_transform.rb b/app/lib/activitypub/case_transform.rb index 93b4b988879..9977647ce0f 100644 --- a/app/lib/activitypub/case_transform.rb +++ b/app/lib/activitypub/case_transform.rb @@ -14,8 +14,7 @@ module ActivityPub::CaseTransform when String camel_lower_cache[value] ||= if value.start_with?('_:') "_:#{value.delete_prefix('_:').underscore.camelize(:lower)}" - elsif LanguagesHelper::SUPPORTED_LOCALES.key?(value.to_sym) || - LanguagesHelper::UI_ONLY_REGIONAL_LOCALES.key?(value.to_sym) + elsif LanguagesHelper::KNOWN_LOCALES.key?(value.to_sym) value else value.underscore.camelize(:lower) diff --git a/lib/tasks/repo.rake b/lib/tasks/repo.rake index e5dfb570d4f..f14049ea0ea 100644 --- a/lib/tasks/repo.rake +++ b/lib/tasks/repo.rake @@ -96,7 +96,7 @@ namespace :repo do end.uniq.compact missing_available_locales = locales_in_files - I18n.available_locales - supported_locale_codes = Set.new(LanguagesHelper::SUPPORTED_LOCALES.keys + LanguagesHelper::UI_ONLY_REGIONAL_LOCALES.keys) + supported_locale_codes = Set.new(LanguagesHelper::KNOWN_LOCALES.keys) missing_locale_names = I18n.available_locales.reject { |locale| supported_locale_codes.include?(locale) } critical = false diff --git a/spec/helpers/languages_helper_spec.rb b/spec/helpers/languages_helper_spec.rb index eae54b61a46..4f1311f8199 100644 --- a/spec/helpers/languages_helper_spec.rb +++ b/spec/helpers/languages_helper_spec.rb @@ -5,7 +5,7 @@ require 'rails_helper' describe LanguagesHelper do describe 'the SUPPORTED_LOCALES constant' do it 'includes all i18n locales' do - expect(Set.new(described_class::SUPPORTED_LOCALES.keys + described_class::UI_ONLY_REGIONAL_LOCALES.keys)).to include(*I18n.available_locales) + expect(Set.new(described_class::KNOWN_LOCALES.keys)).to include(*I18n.available_locales) end end