diff --git a/core/code_systems/serializers.py b/core/code_systems/serializers.py index 8b46cff1e..092ce1a86 100644 --- a/core/code_systems/serializers.py +++ b/core/code_systems/serializers.py @@ -10,7 +10,7 @@ from core.code_systems.constants import RESOURCE_TYPE from core.common.constants import HEAD from core.common.fhir_helpers import delete_empty_fields -from core.common.serializers import ReadSerializerMixin, StatusField, IdentifierSerializer +from core.common.serializers import ReadSerializerMixin, StatusField, IdentifierSerializer, LanguageTagField from core.concepts.models import Concept, ConceptName from core.concepts.serializers import ConceptDetailSerializer from core.fhir import search as fhir_search @@ -34,7 +34,7 @@ def to_representation(self, value): class CodeSystemConceptDesignationSerializer(serializers.ModelSerializer): - language = CharField(source='locale') + language = LanguageTagField(source='locale') value = CharField(source='name') use = CodeSystemConceptDesignationUseSerializer(source='*', required=False) @@ -180,7 +180,7 @@ class CodeSystemDetailSerializer(FhirRepositorySerializerMixin, serializers.Mode url = CanonicalUrlField(source='canonical_url', required=False) title = TitleField(source='full_name', required=False) status = StatusField(source='*') - language = CharField(source='default_locale', required=False) + language = LanguageTagField(source='default_locale', required=False) count = SerializerMethodField(method_name='get_total_count') content = ChoiceField( source='content_type', diff --git a/core/code_systems/tests.py b/core/code_systems/tests.py index 2dc64df63..1c5b9e24a 100644 --- a/core/code_systems/tests.py +++ b/core/code_systems/tests.py @@ -171,6 +171,23 @@ def test_get_code_system(self): self.assertEqual(response.data['id'], self.org_source.mnemonic) self.assertEqual(response.data['version'], 'v2') + def test_languages_are_bcp_47_tags_in_fhir_only(self): + """ + OCL's pt_BR is pt-BR in the CodeSystem's language and its designations; OCL's own API still shows the locale as + it's stored (OpenConceptLab/ocl_issues#3031). + """ + Source.objects.filter(id=self.org_source_v2.id).update(default_locale='pt_BR') + concept = self.org_source_v2.concepts.get(mnemonic=self.concept_1.mnemonic) + concept.names.create(name='Cólera', locale='pt_BR', locale_preferred=True) + + resource = self.client.get(f'/orgs/{self.org.mnemonic}/CodeSystem/{self.org_source.mnemonic}/').data + self.assertEqual(resource['language'], 'pt-BR') + designations = next(item for item in resource['concept'] if item['code'] == concept.mnemonic)['designation'] + self.assertIn({'language': 'pt-BR', 'value': 'Cólera'}, designations) + + self.assertEqual(self.client.get(self.org_source_v2.uri).data['default_locale'], 'pt_BR') + self.assertIn('pt_BR', [name['locale'] for name in self.client.get(concept.uri).data['names']]) + def valid(self, concept): """$validate-code's answer for a valid code of the org source, at the version resolved (v2).""" return {'resourceType': 'Parameters', 'parameter': [ diff --git a/core/common/fhir_helpers.py b/core/common/fhir_helpers.py index fc7218354..d67d96e85 100644 --- a/core/common/fhir_helpers.py +++ b/core/common/fhir_helpers.py @@ -5,6 +5,19 @@ REPOSITORY_PATH = re.compile(r'/(orgs|users)/([^/]+)/([^/]+)/([^/]+)/?') +def bcp47(locale): + """ + An OCL locale as the BCP-47 language tag FHIR needs: OCL, after CIEL, writes pt_BR for pt-BR. Only the separator + changes; locales across the rest of OCL are OpenConceptLab/ocl_issues#2795. + """ + return locale.replace('_', '-') if locale else locale + + +def language_key(locale): + """A locale or language tag as languages are compared: in either spelling, ignoring case, as BCP-47 does.""" + return bcp47(locale).lower() if locale else locale + + def delete_empty_fields(obj): if isinstance(obj, dict): for field in list(obj.keys()): diff --git a/core/common/serializers.py b/core/common/serializers.py index ebadef79d..a206bc9ce 100644 --- a/core/common/serializers.py +++ b/core/common/serializers.py @@ -8,7 +8,7 @@ from core.common.constants import INCLUDE_CONCEPTS_PARAM, INCLUDE_MAPPINGS_PARAM, LIMIT_PARAM, OFFSET_PARAM, \ INCLUDE_VERBOSE_REFERENCES, INCLUDE_SEARCH_META_PARAM, ACCESS_TYPE_CHOICES, ACCESS_TYPE_VIEW from core.common.feeds import DEFAULT_LIMIT -from core.common.fhir_helpers import to_fhir_path, to_ocl_path +from core.common.fhir_helpers import to_fhir_path, to_ocl_path, bcp47 from core.common.utils import to_int, get_truthy_values, normalize_public_access, cap_page_size from core.concept_maps.constants import RESOURCE_TYPE as CONCEPT_MAP_RESOURCE_TYPE from core.orgs.models import Organization @@ -68,6 +68,13 @@ def to_representation(self, value): return 'draft' +class LanguageTagField(CharField): + """An OCL locale as a FHIR resource shows it, a BCP-47 language tag (pt_BR as pt-BR); written as it's given.""" + + def to_representation(self, value): + return bcp47(super().to_representation(value)) + + class IdentifierTypeCodingSerializer(ReadSerializerMixin, Serializer): system = CharField(default='http://hl7.org/fhir/v2/0203', required=False) code = CharField(default='ACSN', required=False) diff --git a/core/concept_maps/serializers.py b/core/concept_maps/serializers.py index 0f6ca8d5d..c9d043d2d 100644 --- a/core/concept_maps/serializers.py +++ b/core/concept_maps/serializers.py @@ -8,7 +8,7 @@ from core.common.fhir_helpers import delete_empty_fields from core.concept_maps.constants import RESOURCE_TYPE, RELATIONSHIP_MAP_TYPES from core.common.constants import HEAD -from core.common.serializers import StatusField, IdentifierSerializer +from core.common.serializers import StatusField, IdentifierSerializer, LanguageTagField from core.fhir.adapters import r4 from core.fhir.ops.translate import concept_map_groups, relationship from core.fhir.resolve import FhirResolver @@ -64,7 +64,7 @@ class ConceptMapDetailSerializer(FhirRepositorySerializerMixin, serializers.Mode url = CanonicalUrlField(source='canonical_url', required=False) title = CharField(source='full_name', required=False) status = StatusField(source='*') - language = CharField(source='default_locale', required=False) + language = LanguageTagField(source='default_locale', required=False) meta = SerializerMethodField() identifier = IdentifierSerializer(many=True, required=False) date = DateTimeField(source='revision_date', required=False) diff --git a/core/concept_maps/tests.py b/core/concept_maps/tests.py index e1ff2e2d8..5ab9d4385 100644 --- a/core/concept_maps/tests.py +++ b/core/concept_maps/tests.py @@ -160,6 +160,13 @@ def test_get_concept_map(self): self.assertEqual(response.data['id'], self.org_source.mnemonic) self.assertEqual(response.data['version'], 'v2') + def test_language_is_a_bcp_47_tag(self): + """OCL's pt_BR is pt-BR in FHIR (OpenConceptLab/ocl_issues#3031).""" + Source.objects.filter(id=self.org_source_v2.id).update(default_locale='pt_BR') + + response = self.client.get(f'/orgs/{self.org.mnemonic}/ConceptMap/{self.org_source.mnemonic}/') + self.assertEqual(response.data['language'], 'pt-BR') + @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) def test_post_concept_map_without_mappings(self): diff --git a/core/fhir/ops/concepts.py b/core/fhir/ops/concepts.py index 143d782fe..7b5412818 100644 --- a/core/fhir/ops/concepts.py +++ b/core/fhir/ops/concepts.py @@ -8,14 +8,16 @@ - A code is found among the version's own concepts, retired ones included, as FHIR spells it or percent-encoded, as OCL stores a code with reserved characters. One query for the concept (with its repository), one for its names. - Display: with a language, the concept's preferred name in that language when it has one; else its own display (OCL's - display_name: the preferred name in its code system's default locale). + display_name: the preferred name in its code system's default locale). A language matches a locale in either + spelling (pt-BR, OCL's pt_BR), ignoring case. +- Designations: each of the concept's names, its locale as a BCP-47 tag (pt_BR as pt-BR). """ from urllib.parse import quote, unquote from django.db.models import F, Q from core.common.constants import HEAD -from core.common.fhir_helpers import to_ocl_path +from core.common.fhir_helpers import bcp47, language_key, to_ocl_path from core.common.utils import to_parent_uri from core.concepts.models import Concept from core.fhir import outcomes @@ -131,11 +133,12 @@ def find_concept(version, code): def display_of(concept, language=None): """ - The concept's display: with a language, its preferred name in that language, newest first, when it has one; else - its own display. None for a concept without names. + The concept's display: with a language, its preferred name in that language (either spelling, any case), newest + first, when it has one; else its own display. None for a concept without names. """ if language: - names = sorted((name for name in concept.names.all() if name.locale == language and not name.retired), + key = language_key(language) + names = sorted((name for name in concept.names.all() if language_key(name.locale) == key and not name.retired), key=lambda name: (name.locale_preferred, name.created_at), reverse=True) if names: return names[0].name @@ -152,6 +155,6 @@ def has_name(concept, display): def designations(concept): - """Each of the concept's names as a designation: its language, its OCL name type as `use`, and its text.""" - return [{'language': name.locale, 'use': {'code': name.type} if name.type else None, 'value': name.name} + """Each of the concept's names as a designation: its language tag, its OCL name type as `use`, and its text.""" + return [{'language': bcp47(name.locale), 'use': {'code': name.type} if name.type else None, 'value': name.name} for name in concept.names.all()] diff --git a/core/fhir/search.py b/core/fhir/search.py index fffce1964..207ad20e5 100644 --- a/core/fhir/search.py +++ b/core/fhir/search.py @@ -37,7 +37,7 @@ from django.conf import settings from django.db import connection, transaction, OperationalError from django.db.models import BooleanField, Case, F, Exists, Func, OuterRef, Q, Subquery, TextField, Value, When -from django.db.models.functions import Coalesce +from django.db.models.functions import Coalesce, Lower, Replace from django.db.models.lookups import Exact, IContains, In, IStartsWith from django.utils import timezone from psycopg2 import errorcodes @@ -46,7 +46,7 @@ from core.collections.models import Expansion from core.common.constants import HEAD from core.common.db_functions import ImmutableUnaccent -from core.common.fhir_helpers import repository_prefixes +from core.common.fhir_helpers import language_key, repository_prefixes from core.common.mixins import CustomPaginator from core.common.utils import to_int from core.concepts.models import Concept @@ -515,7 +515,9 @@ def by_code(value): @staticmethod def by_language(value): - return Q(default_locale=value) + """The language CodeSystem.language shows, a BCP-47 tag: a default locale in either spelling, any case.""" + locale = Replace('default_locale', Value('_'), Value('-'), output_field=TextField()) + return Q(Exact(Lower(locale), language_key(value))) def by_reference(self, value): """ diff --git a/core/fhir/tests_ops_expand.py b/core/fhir/tests_ops_expand.py index fd1bce3ea..f4f0d665b 100644 --- a/core/fhir/tests_ops_expand.py +++ b/core/fhir/tests_ops_expand.py @@ -22,7 +22,7 @@ from core.common.constants import ACCESS_TYPE_NONE from core.common.tests import OCLTestCase, OCLAPITransactionTestCase from core.concepts.models import Concept -from core.concepts.tests.factories import ConceptFactory +from core.concepts.tests.factories import ConceptFactory, ConceptNameFactory from core.fhir.outcomes import MESSAGE_ID_EXTENSION, TX_ISSUE_TYPE_SYSTEM from core.fhir.tests_ops_lookup import CIEL, FEVERS, HIV, NEWEST_PRIVATE, OpsCorpusMixin, REST, name from core.fhir.tests_search import collection, source @@ -197,6 +197,17 @@ def test_display_language_and_designations(self): {'name': 'displayLanguage', 'valueCode': 'fr'}, {'name': 'includeDesignations', 'valueBoolean': True}]) self.assertNotIn('designation', self.expansion(self.expand(url=FEVERS))['contains'][0]) + def test_languages_are_bcp_47_tags_and_display_language_takes_either_spelling(self): + """OCL's pt_BR is pt-BR in FHIR; displayLanguage finds it by either spelling, in any case (ocl_issues#3031).""" + ConceptNameFactory(concept=self.infectious, name='Enterite infecciosa', locale='pt_BR', locale_preferred=True) + + for language in ('pt-BR', 'pt_br'): + expansion = self.expansion( + self.expand(url=FEVERS, displayLanguage=language, includeDesignations='true', count='1')) + self.assertEqual(expansion['contains'][0]['display'], 'Enterite infecciosa', language) + self.assertEqual( + sorted({item['language'] for item in expansion['contains'][0]['designation']}), ['en', 'es', 'fr', 'pt-BR']) + # versions: only releases the caller is shown def test_a_members_version_is_the_release_the_expansion_pins_then_evaluated_then_newest(self): diff --git a/core/fhir/tests_ops_lookup.py b/core/fhir/tests_ops_lookup.py index 7aad2b665..f6751a337 100644 --- a/core/fhir/tests_ops_lookup.py +++ b/core/fhir/tests_ops_lookup.py @@ -216,6 +216,18 @@ def test_display_language(self): response = self.lookup(system=CIEL, code='117053', displayLanguage='ru') self.assertEqual(self.by_name(response)['display']['valueString'], 'Infectious enteritis') + def test_languages_are_bcp_47_tags_and_display_language_takes_either_spelling(self): + """OCL's pt_BR is pt-BR in FHIR; displayLanguage finds it by either spelling, in any case (ocl_issues#3031).""" + ConceptNameFactory(concept=self.infectious, name='Enterite infecciosa', locale='pt_BR', locale_preferred=True) + + for language in ('pt-BR', 'pt_BR', 'PT-br'): + response = self.lookup(system=CIEL, code='117053', displayLanguage=language) + self.assertEqual(self.by_name(response)['display']['valueString'], 'Enterite infecciosa', language) + languages = {part['valueCode'] for designation in self.by_name(response)['designations'] + for part in designation['part'] if part['name'] == 'language'} + self.assertEqual(languages, {'en', 'es', 'fr', 'pt-BR'}) + self.assertEqual(self.infectious.names.get(name='Enterite infecciosa').locale, 'pt_BR') # stored as it was + def test_a_concept_without_names_is_displayed_by_its_code(self): response = self.lookup(system=REST, code='batch') diff --git a/core/fhir/tests_search.py b/core/fhir/tests_search.py index 86b8133a2..513c09a93 100644 --- a/core/fhir/tests_search.py +++ b/core/fhir/tests_search.py @@ -407,6 +407,13 @@ def test_identifier_system_is_matched_on_the_same_identifier(self): Source.objects.filter(id=self.ciel_v1.id).update(identifier={'system': 'urn:x', 'value': 'y'}) self.assertEqual(self.search('CodeSystem', identifier='urn:x|')['total'], 0) + def test_language_takes_either_spelling(self): + """CodeSystem.language shows OCL's pt_BR as pt-BR: both find it, in any case (ocl_issues#3031).""" + Source.objects.filter(id=self.ciel_v2.id).update(default_locale='pt_BR') + for language in ('pt-BR', 'pt_BR', 'PT-br', 'urn:ietf:bcp:47|pt-BR'): + self.assertEqual(self.keys(self.search('CodeSystem', language=language)), [f'{CIEL}|v2'], language) + self.assertEqual(self.search('CodeSystem', language='pt')['total'], 0) + def test_language_system_only_needs_a_language(self): Source.objects.filter(id=self.ciel_v2.id).update(default_locale='') self.assertEqual(self.keys(self.search('CodeSystem', language='urn:ietf:bcp:47|')), sorted(