OpenConceptLab/ocl_issues#3031 | FHIR: send OCL locales as BCP-47 language tags

core/code_systems/serializers.py

@@ -10,7 +10,7 @@
 from core.code_systems.constants import RESOURCE_TYPE
 from core.common.constants import HEAD
 from core.common.fhir_helpers import delete_empty_fields
-from core.common.serializers import ReadSerializerMixin, StatusField, IdentifierSerializer
+from core.common.serializers import ReadSerializerMixin, StatusField, IdentifierSerializer, LanguageTagField
 from core.concepts.models import Concept, ConceptName
 from core.concepts.serializers import ConceptDetailSerializer
 from core.fhir import search as fhir_search
@@ -34,7 +34,7 @@ def to_representation(self, value):
 
 
 class CodeSystemConceptDesignationSerializer(serializers.ModelSerializer):
-    language = CharField(source='locale')
+    language = LanguageTagField(source='locale')
     value = CharField(source='name')
     use = CodeSystemConceptDesignationUseSerializer(source='*', required=False)
 
@@ -180,7 +180,7 @@ class CodeSystemDetailSerializer(FhirRepositorySerializerMixin, serializers.Mode
     url = CanonicalUrlField(source='canonical_url', required=False)
     title = TitleField(source='full_name', required=False)
     status = StatusField(source='*')
-    language = CharField(source='default_locale', required=False)
+    language = LanguageTagField(source='default_locale', required=False)
     count = SerializerMethodField(method_name='get_total_count')
     content = ChoiceField(
         source='content_type',

core/code_systems/tests.py

@@ -171,6 +171,23 @@ def test_get_code_system(self):
         self.assertEqual(response.data['id'], self.org_source.mnemonic)
         self.assertEqual(response.data['version'], 'v2')
 
+    def test_languages_are_bcp_47_tags_in_fhir_only(self):
+        """
+        OCL's pt_BR is pt-BR in the CodeSystem's language and its designations; OCL's own API still shows the locale as
+        it's stored (OpenConceptLab/ocl_issues#3031).
+        """
+        Source.objects.filter(id=self.org_source_v2.id).update(default_locale='pt_BR')
+        concept = self.org_source_v2.concepts.get(mnemonic=self.concept_1.mnemonic)
+        concept.names.create(name='Cólera', locale='pt_BR', locale_preferred=True)
+
+        resource = self.client.get(f'/orgs/{self.org.mnemonic}/CodeSystem/{self.org_source.mnemonic}/').data
+        self.assertEqual(resource['language'], 'pt-BR')
+        designations = next(item for item in resource['concept'] if item['code'] == concept.mnemonic)['designation']
+        self.assertIn({'language': 'pt-BR', 'value': 'Cólera'}, designations)
+
+        self.assertEqual(self.client.get(self.org_source_v2.uri).data['default_locale'], 'pt_BR')
+        self.assertIn('pt_BR', [name['locale'] for name in self.client.get(concept.uri).data['names']])
+
     def valid(self, concept):
         """$validate-code's answer for a valid code of the org source, at the version resolved (v2)."""
         return {'resourceType': 'Parameters', 'parameter': [

core/common/fhir_helpers.py

@@ -5,6 +5,19 @@
 REPOSITORY_PATH = re.compile(r'/(orgs|users)/([^/]+)/([^/]+)/([^/]+)/?')
 
 
+def bcp47(locale):
+    """
+    An OCL locale as the BCP-47 language tag FHIR needs: OCL, after CIEL, writes pt_BR for pt-BR. Only the separator
+    changes; locales across the rest of OCL are OpenConceptLab/ocl_issues#2795.
+    """
+    return locale.replace('_', '-') if locale else locale
+
+
+def language_key(locale):
+    """A locale or language tag as languages are compared: in either spelling, ignoring case, as BCP-47 does."""
+    return bcp47(locale).lower() if locale else locale
+
+
 def delete_empty_fields(obj):
     if isinstance(obj, dict):
         for field in list(obj.keys()):

core/common/serializers.py

@@ -8,7 +8,7 @@
 from core.common.constants import INCLUDE_CONCEPTS_PARAM, INCLUDE_MAPPINGS_PARAM, LIMIT_PARAM, OFFSET_PARAM, \
     INCLUDE_VERBOSE_REFERENCES, INCLUDE_SEARCH_META_PARAM, ACCESS_TYPE_CHOICES, ACCESS_TYPE_VIEW
 from core.common.feeds import DEFAULT_LIMIT
-from core.common.fhir_helpers import to_fhir_path, to_ocl_path
+from core.common.fhir_helpers import to_fhir_path, to_ocl_path, bcp47
 from core.common.utils import to_int, get_truthy_values, normalize_public_access, cap_page_size
 from core.concept_maps.constants import RESOURCE_TYPE as CONCEPT_MAP_RESOURCE_TYPE
 from core.orgs.models import Organization
@@ -68,6 +68,13 @@ def to_representation(self, value):
         return 'draft'
 
 
+class LanguageTagField(CharField):
+    """An OCL locale as a FHIR resource shows it, a BCP-47 language tag (pt_BR as pt-BR); written as it's given."""
+
+    def to_representation(self, value):
+        return bcp47(super().to_representation(value))
+
+
 class IdentifierTypeCodingSerializer(ReadSerializerMixin, Serializer):
     system = CharField(default='http://hl7.org/fhir/v2/0203', required=False)
     code = CharField(default='ACSN', required=False)

core/concept_maps/serializers.py

@@ -8,7 +8,7 @@
 from core.common.fhir_helpers import delete_empty_fields
 from core.concept_maps.constants import RESOURCE_TYPE, RELATIONSHIP_MAP_TYPES
 from core.common.constants import HEAD
-from core.common.serializers import StatusField, IdentifierSerializer
+from core.common.serializers import StatusField, IdentifierSerializer, LanguageTagField
 from core.fhir.adapters import r4
 from core.fhir.ops.translate import concept_map_groups, relationship
 from core.fhir.resolve import FhirResolver
@@ -64,7 +64,7 @@ class ConceptMapDetailSerializer(FhirRepositorySerializerMixin, serializers.Mode
     url = CanonicalUrlField(source='canonical_url', required=False)
     title = CharField(source='full_name', required=False)
     status = StatusField(source='*')
-    language = CharField(source='default_locale', required=False)
+    language = LanguageTagField(source='default_locale', required=False)
     meta = SerializerMethodField()
     identifier = IdentifierSerializer(many=True, required=False)
     date = DateTimeField(source='revision_date', required=False)

core/concept_maps/tests.py

@@ -160,6 +160,13 @@ def test_get_concept_map(self):
         self.assertEqual(response.data['id'], self.org_source.mnemonic)
         self.assertEqual(response.data['version'], 'v2')
 
+    def test_language_is_a_bcp_47_tag(self):
+        """OCL's pt_BR is pt-BR in FHIR (OpenConceptLab/ocl_issues#3031)."""
+        Source.objects.filter(id=self.org_source_v2.id).update(default_locale='pt_BR')
+
+        response = self.client.get(f'/orgs/{self.org.mnemonic}/ConceptMap/{self.org_source.mnemonic}/')
+        self.assertEqual(response.data['language'], 'pt-BR')
+
     @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings'))
     @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts'))
     def test_post_concept_map_without_mappings(self):

core/fhir/ops/concepts.py

@@ -8,14 +8,16 @@
 - A code is found among the version's own concepts, retired ones included, as FHIR spells it or percent-encoded, as
   OCL stores a code with reserved characters. One query for the concept (with its repository), one for its names.
 - Display: with a language, the concept's preferred name in that language when it has one; else its own display (OCL's
-  display_name: the preferred name in its code system's default locale).
+  display_name: the preferred name in its code system's default locale). A language matches a locale in either
+  spelling (pt-BR, OCL's pt_BR), ignoring case.
+- Designations: each of the concept's names, its locale as a BCP-47 tag (pt_BR as pt-BR).
 """
 from urllib.parse import quote, unquote
 
 from django.db.models import F, Q
 
 from core.common.constants import HEAD
-from core.common.fhir_helpers import to_ocl_path
+from core.common.fhir_helpers import bcp47, language_key, to_ocl_path
 from core.common.utils import to_parent_uri
 from core.concepts.models import Concept
 from core.fhir import outcomes
@@ -131,11 +133,12 @@ def find_concept(version, code):
 
 def display_of(concept, language=None):
     """
-    The concept's display: with a language, its preferred name in that language, newest first, when it has one; else
-    its own display. None for a concept without names.
+    The concept's display: with a language, its preferred name in that language (either spelling, any case), newest
+    first, when it has one; else its own display. None for a concept without names.
     """
     if language:
-        names = sorted((name for name in concept.names.all() if name.locale == language and not name.retired),
+        key = language_key(language)
+        names = sorted((name for name in concept.names.all() if language_key(name.locale) == key and not name.retired),
                        key=lambda name: (name.locale_preferred, name.created_at), reverse=True)
         if names:
             return names[0].name
@@ -152,6 +155,6 @@ def has_name(concept, display):
 
 
 def designations(concept):
-    """Each of the concept's names as a designation: its language, its OCL name type as `use`, and its text."""
-    return [{'language': name.locale, 'use': {'code': name.type} if name.type else None, 'value': name.name}
+    """Each of the concept's names as a designation: its language tag, its OCL name type as `use`, and its text."""
+    return [{'language': bcp47(name.locale), 'use': {'code': name.type} if name.type else None, 'value': name.name}
             for name in concept.names.all()]

core/fhir/search.py

@@ -37,7 +37,7 @@
 from django.conf import settings
 from django.db import connection, transaction, OperationalError
 from django.db.models import BooleanField, Case, F, Exists, Func, OuterRef, Q, Subquery, TextField, Value, When
-from django.db.models.functions import Coalesce
+from django.db.models.functions import Coalesce, Lower, Replace
 from django.db.models.lookups import Exact, IContains, In, IStartsWith
 from django.utils import timezone
 from psycopg2 import errorcodes
@@ -46,7 +46,7 @@
 from core.collections.models import Expansion
 from core.common.constants import HEAD
 from core.common.db_functions import ImmutableUnaccent
-from core.common.fhir_helpers import repository_prefixes
+from core.common.fhir_helpers import language_key, repository_prefixes
 from core.common.mixins import CustomPaginator
 from core.common.utils import to_int
 from core.concepts.models import Concept
@@ -515,7 +515,9 @@ def by_code(value):
 
     @staticmethod
     def by_language(value):
-        return Q(default_locale=value)
+        """The language CodeSystem.language shows, a BCP-47 tag: a default locale in either spelling, any case."""
+        locale = Replace('default_locale', Value('_'), Value('-'), output_field=TextField())
+        return Q(Exact(Lower(locale), language_key(value)))
 
     def by_reference(self, value):
         """

core/fhir/tests_ops_expand.py

@@ -22,7 +22,7 @@
 from core.common.constants import ACCESS_TYPE_NONE
 from core.common.tests import OCLTestCase, OCLAPITransactionTestCase
 from core.concepts.models import Concept
-from core.concepts.tests.factories import ConceptFactory
+from core.concepts.tests.factories import ConceptFactory, ConceptNameFactory
 from core.fhir.outcomes import MESSAGE_ID_EXTENSION, TX_ISSUE_TYPE_SYSTEM
 from core.fhir.tests_ops_lookup import CIEL, FEVERS, HIV, NEWEST_PRIVATE, OpsCorpusMixin, REST, name
 from core.fhir.tests_search import collection, source
@@ -197,6 +197,17 @@ def test_display_language_and_designations(self):
             {'name': 'displayLanguage', 'valueCode': 'fr'}, {'name': 'includeDesignations', 'valueBoolean': True}])
         self.assertNotIn('designation', self.expansion(self.expand(url=FEVERS))['contains'][0])
 
+    def test_languages_are_bcp_47_tags_and_display_language_takes_either_spelling(self):
+        """OCL's pt_BR is pt-BR in FHIR; displayLanguage finds it by either spelling, in any case (ocl_issues#3031)."""
+        ConceptNameFactory(concept=self.infectious, name='Enterite infecciosa', locale='pt_BR', locale_preferred=True)
+
+        for language in ('pt-BR', 'pt_br'):
+            expansion = self.expansion(
+                self.expand(url=FEVERS, displayLanguage=language, includeDesignations='true', count='1'))
+            self.assertEqual(expansion['contains'][0]['display'], 'Enterite infecciosa', language)
+        self.assertEqual(
+            sorted({item['language'] for item in expansion['contains'][0]['designation']}), ['en', 'es', 'fr', 'pt-BR'])
+
     # versions: only releases the caller is shown
 
     def test_a_members_version_is_the_release_the_expansion_pins_then_evaluated_then_newest(self):

core/fhir/tests_ops_lookup.py

@@ -216,6 +216,18 @@ def test_display_language(self):
         response = self.lookup(system=CIEL, code='117053', displayLanguage='ru')
         self.assertEqual(self.by_name(response)['display']['valueString'], 'Infectious enteritis')
 
+    def test_languages_are_bcp_47_tags_and_display_language_takes_either_spelling(self):
+        """OCL's pt_BR is pt-BR in FHIR; displayLanguage finds it by either spelling, in any case (ocl_issues#3031)."""
+        ConceptNameFactory(concept=self.infectious, name='Enterite infecciosa', locale='pt_BR', locale_preferred=True)
+
+        for language in ('pt-BR', 'pt_BR', 'PT-br'):
+            response = self.lookup(system=CIEL, code='117053', displayLanguage=language)
+            self.assertEqual(self.by_name(response)['display']['valueString'], 'Enterite infecciosa', language)
+        languages = {part['valueCode'] for designation in self.by_name(response)['designations']
+                     for part in designation['part'] if part['name'] == 'language'}
+        self.assertEqual(languages, {'en', 'es', 'fr', 'pt-BR'})
+        self.assertEqual(self.infectious.names.get(name='Enterite infecciosa').locale, 'pt_BR')  # stored as it was
+
     def test_a_concept_without_names_is_displayed_by_its_code(self):
         response = self.lookup(system=REST, code='batch')
 

core/fhir/tests_search.py

@@ -407,6 +407,13 @@ def test_identifier_system_is_matched_on_the_same_identifier(self):
         Source.objects.filter(id=self.ciel_v1.id).update(identifier={'system': 'urn:x', 'value': 'y'})
         self.assertEqual(self.search('CodeSystem', identifier='urn:x|')['total'], 0)
 
+    def test_language_takes_either_spelling(self):
+        """CodeSystem.language shows OCL's pt_BR as pt-BR: both find it, in any case (ocl_issues#3031)."""
+        Source.objects.filter(id=self.ciel_v2.id).update(default_locale='pt_BR')
+        for language in ('pt-BR', 'pt_BR', 'PT-br', 'urn:ietf:bcp:47|pt-BR'):
+            self.assertEqual(self.keys(self.search('CodeSystem', language=language)), [f'{CIEL}|v2'], language)
+        self.assertEqual(self.search('CodeSystem', language='pt')['total'], 0)
+
     def test_language_system_only_needs_a_language(self):
         Source.objects.filter(id=self.ciel_v2.id).update(default_locale='')
         self.assertEqual(self.keys(self.search('CodeSystem', language='urn:ietf:bcp:47|')), sorted(