OpenConceptLab/ocl_issues#3030 | Sources: store and publish which code system a supplement supplements

core/code_systems/constants.py

@@ -1 +1,5 @@
 RESOURCE_TYPE = 'CodeSystem'
+SUPPLEMENT_TYPE_EXTENSION = 'http://hl7.org/fhir/StructureDefinition/codesystem-supplement-type'
+SUPPLEMENT_GLOBAL_EXTENSION = 'http://hl7.org/fhir/StructureDefinition/codesystem-supplement-global'
+SUPPLEMENT_CONCEPT_NEEDS_A_NAME = 'A supplement concept needs a display or a designation: OCL keeps a concept only ' \
+                                  'with a name.'

core/code_systems/serializers.py

@@ -2,19 +2,22 @@
 import logging
 from collections import OrderedDict
 
+from pydash import get
 from rest_framework import serializers
 from rest_framework.fields import CharField, BooleanField, SerializerMethodField, ChoiceField, \
     DateTimeField
 
 from core import settings
-from core.code_systems.constants import RESOURCE_TYPE
+from core.code_systems.constants import RESOURCE_TYPE, SUPPLEMENT_TYPE_EXTENSION, SUPPLEMENT_GLOBAL_EXTENSION, \
+    SUPPLEMENT_CONCEPT_NEEDS_A_NAME
 from core.common.constants import HEAD
 from core.common.fhir_helpers import delete_empty_fields
 from core.common.serializers import ReadSerializerMixin, StatusField, IdentifierSerializer
 from core.concepts.models import Concept, ConceptName
 from core.concepts.serializers import ConceptDetailSerializer
 from core.fhir import search as fhir_search
 from core.fhir.serializers import CanonicalUrlField, FhirRepositorySerializerMixin, TitleField
+from core.sources.constants import CONTENT_TYPE_SUPPLEMENT
 from core.sources.models import Source
 from core.sources.serializers import SourceCreateOrUpdateSerializer
 
@@ -111,18 +114,47 @@ def to_internal_value(self, data):
         if 'names' not in ret:
             ret.update({'names': []})
 
-        found = False
-        for concept_name in ret['names']:
-            if concept_name['name'] == data['display'] and concept_name['locale'] == settings.DEFAULT_LOCALE:
+        display = data.get('display')
+        if self.context.get('supplement'):  # nothing made up: a display is the name in the code system's language
+            self.add_preferred_name(ret['names'], display, self.context['language'])
+            if not ret['names']:  # refused here, before anything is saved: the concept wouldn't be
+                raise serializers.ValidationError({'designation': [SUPPLEMENT_CONCEPT_NEEDS_A_NAME]})
+        elif display or not ret['names']:  # a concept needs a name: the code, when it has no other
+            self.add_preferred_name(ret['names'], display or ret['mnemonic'], settings.DEFAULT_LOCALE)
+
+        return ret
+
+    @staticmethod
+    def add_preferred_name(names, name, locale):
+        """`name` as the preferred name in `locale`: the designation with that value there, else a new name."""
+        if not name:
+            return
+        for concept_name in names:
+            if concept_name['name'] == name and concept_name['locale'] == locale:
                 concept_name['locale_preferred'] = True
-                found = True
-                break
+                return
+        names.append({'name': name, 'locale': locale, 'locale_preferred': True})
 
-        if not found:
-            ret['names'].append({'name': data.get('display', data.get('code', None)), 'locale': settings.DEFAULT_LOCALE,
-                                 'locale_preferred': True})
 
-        return ret
+class CodeSystemSupplementConceptSerializer(CodeSystemConceptSerializer):
+    """
+    A supplement's concept has only what the supplement adds to the code: its names, and inactive when retired (no
+    concept class or datatype, which are the supplemented code system's). Its preferred name in the code system's
+    language (context language) is its display, FHIR's primary designation, and is repeated as a designation only
+    when it has a use of its own.
+    """
+    def to_representation(self, instance):
+        names = list(instance.names.all())
+        display = next((name for name in names if name.locale_preferred and not name.retired and
+                        name.locale == self.context['language']), None)
+        return OrderedDict([
+            ('code', instance.mnemonic),
+            ('display', display.name if display else None),
+            ('definition', self.fields['definition'].to_representation(instance)),
+            ('designation', CodeSystemConceptDesignationSerializer(
+                [name for name in names if name is not display or name.type], many=True).data),
+            ('property', [{'code': 'inactive', 'valueBoolean': True}] if instance.retired else []),
+        ])
 
 
 class CodeSystemPropertySerializer(ReadSerializerMixin, serializers.Serializer):
@@ -143,14 +175,48 @@ def listed_concepts(context, source):
 
 class CodeSystemConceptField(serializers.Field):
     def to_internal_value(self, data):
-        concepts = CodeSystemConceptSerializer(data=data, many=True)
+        """The concepts written, as the code system they're written to says: its content and its language."""
+        code_system, instance = self.parent.initial_data, self.parent.instance
+        content = code_system.get('content', get(instance, 'content_type'))
+        language = code_system.get('language') or get(instance, 'default_locale') or settings.DEFAULT_LOCALE
+        concepts = CodeSystemConceptSerializer(
+            data=data, many=True, context={'supplement': content == CONTENT_TYPE_SUPPLEMENT, 'language': language})
         concepts.is_valid(raise_exception=True)
         return {'concepts': concepts.validated_data}
 
     def to_representation(self, value):
-        """Every concept, retired ones included, up to the limit; none above it (content not-present)."""
+        """
+        Every concept, retired ones included, up to the limit; none above it: content not-present, or for a supplement,
+        which stays one, SUBSETTED.
+        """
         concepts = listed_concepts(self.context, value)[1]
-        return CodeSystemConceptSerializer(concepts, many=True).data if concepts else []
+        self.subsetted = concepts is None and value.is_supplement
+        if not concepts:
+            return []
+        if value.is_supplement:
+            return CodeSystemSupplementConceptSerializer(
+                concepts, many=True, context={'language': value.default_locale}).data
+        return CodeSystemConceptSerializer(concepts, many=True).data
+
+
+class CodeSystemExtensionField(serializers.Field):
+    """The extensions OCL keeps: a supplement's codesystem-supplement-type and codesystem-supplement-global."""
+    def to_internal_value(self, data):
+        if not isinstance(data, list):
+            raise serializers.ValidationError('extension must be a list.')
+        extensions = {item.get('url'): item for item in data if isinstance(item, dict)}
+        return {
+            'supplement_type': extensions.get(SUPPLEMENT_TYPE_EXTENSION, {}).get('valueCode'),
+            'supplement_global': extensions.get(SUPPLEMENT_GLOBAL_EXTENSION, {}).get('valueBoolean'),
+        }
+
+    def to_representation(self, value):
+        extensions = []
+        if value.supplement_type:
+            extensions.append({'url': SUPPLEMENT_TYPE_EXTENSION, 'valueCode': value.supplement_type})
+        if value.supplement_global is not None:
+            extensions.append({'url': SUPPLEMENT_GLOBAL_EXTENSION, 'valueBoolean': value.supplement_global})
+        return extensions
 
 
 class TextField(ReadSerializerMixin, serializers.Serializer):
@@ -187,6 +253,8 @@ class CodeSystemDetailSerializer(FhirRepositorySerializerMixin, serializers.Mode
         choices=['not-present', 'example', 'fragment', 'complete', 'supplement'],
         required=True
     )
+    supplements = CharField(required=False)
+    extension = CodeSystemExtensionField(source='*', required=False)
     property = SerializerMethodField()
     meta = SerializerMethodField()
     concept = CodeSystemConceptField(source='*', required=False)
@@ -201,7 +269,8 @@ class CodeSystemDetailSerializer(FhirRepositorySerializerMixin, serializers.Mode
 
     class Meta:
         model = Source
-        fields = ('resourceType', 'url', 'title', 'status', 'id', 'language', 'count', 'content', 'property', 'meta',
+        fields = ('resourceType', 'extension', 'url', 'title', 'status', 'id', 'language', 'count', 'content',
+                  'supplements', 'property', 'meta',
                   'version', 'identifier', 'contact', 'jurisdiction', 'name', 'description', 'publisher', 'purpose',
                   'copyright', 'revisionDate', 'experimental', 'caseSensitive', 'compositional', 'versionNeeded',
                   'collectionReference', 'hierarchyMeaning', 'concept', 'text', 'filters')
@@ -217,8 +286,10 @@ def get_total_count(self, obj):
     def get_content(self, obj):
         """
         complete (or the content mode the CodeSystem was saved with) up to the concept limit, else not-present: a
-        search annotates concepts_over_limit, a read counts them.
+        search annotates concepts_over_limit, a read counts them. A supplement is one at any size.
         """
+        if obj.is_supplement:
+            return CONTENT_TYPE_SUPPLEMENT
         over_limit = getattr(obj, 'concepts_over_limit', None)
         if over_limit is None:
             over_limit = listed_concepts(self.context, obj)[1] is None
@@ -248,6 +319,8 @@ def get_property(obj):
                 'type': 'Coding'
             }
         ]
+        if obj.is_supplement:  # its concepts have no concept class or datatype (CodeSystemSupplementConceptSerializer)
+            properties = [prop for prop in properties if prop['code'] == 'inactive']
         new_properties = [*(obj.properties or [])]
         defined_property_codes = {prop['code'] for prop in new_properties}
         for prop in properties:

core/code_systems/tests.py

@@ -1,5 +1,6 @@
 import json
 
+from django.test import override_settings
 from fhir.resources.R4B.codesystem import CodeSystem
 from mock.mock import patch, Mock
 from rest_framework.test import APIClient
@@ -10,6 +11,7 @@
 from core.common.tests import OCLTestCase
 from core.concepts.models import Concept
 from core.concepts.tests.factories import ConceptFactory
+from core.fhir.serializers import SUBSETTED
 from core.orgs.tests.factories import OrganizationFactory
 from core.sources.models import Source
 from core.sources.tests.factories import OrganizationSourceFactory, UserSourceFactory
@@ -764,3 +766,143 @@ def test_conversion_to_fhir(self):
         self.assertTrue(can_convert)
         code_system = CodeSystemConverter.to_fhir(source)
         self.assertEqual(code_system['id'], 'test')
+
+
+SUPPLEMENTED = 'http://hl7.org/fhir/sid/icd-10|2019-covid-expanded'
+LANG_PACK = [
+    {'url': 'http://hl7.org/fhir/StructureDefinition/codesystem-supplement-type', 'valueCode': 'lang-pack'},
+    {'url': 'http://hl7.org/fhir/StructureDefinition/codesystem-supplement-global', 'valueBoolean': True}]
+
+
+@patch('core.common.tasks.__handle_save', Mock(__name__='__handle_save'))
+@patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts'))
+@patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings'))
+class CodeSystemSupplementTest(OCLTestCase):
+    """
+    A supplement written and read as FHIR shows one, like HL7 Denmark's ICD-10 supplement on tx.fhir.org: content
+    supplement, supplements, and concepts that are a code and their designations (OpenConceptLab/ocl_issues#3030).
+    """
+    def setUp(self):
+        super().setUp()
+        self.user = UserProfileFactory()
+        self.client = APIClient(HTTP_AUTHORIZATION='Token ' + self.user.get_token())
+        self.path = f'/users/{self.user.mnemonic}/CodeSystem/'
+
+    def write(self, method='post', **resource):
+        body = {
+            'resourceType': 'CodeSystem', 'id': 'icd10-fr', 'url': 'http://example.org/CodeSystem/icd10-fr',
+            'version': '1', 'name': 'icd10-fr', 'title': 'ICD-10 fr', 'status': 'active', 'language': 'fr',
+            'content': 'supplement', 'supplements': SUPPLEMENTED, **resource}
+        path = self.path + ('icd10-fr/' if method == 'put' else '')
+        return getattr(self.client, method)(
+            path, {key: value for key, value in body.items() if value is not None}, format='json')
+
+    def read(self):
+        response = self.client.get(self.path + 'icd10-fr/')
+        self.assertEqual(response.status_code, 200, response.data)
+        return response.data
+
+    def head(self):
+        return Source.objects.get(mnemonic='icd10-fr', version='HEAD', user=self.user)
+
+    def names(self, code):
+        concept = self.head().get_concepts_queryset().get(mnemonic=code)
+        return sorted((name.locale, name.name, name.locale_preferred) for name in concept.names.all())
+
+    def test_a_supplement_is_written_and_read_without_anything_added(self):
+        response = self.write(extension=LANG_PACK, concept=[
+            {'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]},
+            {'code': 'A01', 'designation': [{'language': 'fr', 'value': 'Fièvres typhoïde et paratyphoïde'}]}])
+        self.assertEqual(response.status_code, 201, response.data)
+
+        head = self.head()
+        self.assertEqual(
+            [head.content_type, head.supplements, head.supplement_type, head.supplement_global],
+            ['supplement', SUPPLEMENTED, 'lang-pack', True])
+        self.assertEqual(self.names('A00'), [('fr', 'Choléra', False)])  # no name made up from the code
+
+        resource = self.read()
+        self.assertEqual(
+            [resource['content'], resource['supplements'], resource['extension']],
+            ['supplement', SUPPLEMENTED, LANG_PACK])
+        self.assertEqual(
+            resource['concept'][0], {'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]})
+        self.assertEqual([prop['code'] for prop in resource['property']], ['inactive'])
+
+    def test_a_supplements_display_is_its_name_in_the_code_systems_language(self):
+        """FHIR: the display is a designation in the resource's language with the use 'primary'."""
+        response = self.write(concept=[{'code': 'A00', 'display': 'Choléra', 'designation': [
+            {'language': 'fr', 'value': 'Choléra, sans précision'}, {'language': 'en', 'value': 'Cholera'}]}])
+        self.assertEqual(response.status_code, 201, response.data)
+
+        self.assertEqual(
+            self.names('A00'),
+            [('en', 'Cholera', False), ('fr', 'Choléra', True), ('fr', 'Choléra, sans précision', False)])
+        self.assertEqual(self.read()['concept'][0], {
+            'code': 'A00', 'display': 'Choléra', 'designation': [
+                {'language': 'fr', 'value': 'Choléra, sans précision'}, {'language': 'en', 'value': 'Cholera'}]})
+
+    def test_a_designation_with_a_use_is_kept_when_it_is_also_the_display(self):
+        designation = {'language': 'fr', 'value': 'Choléra', 'use': {'code': 'synonym'}}
+        response = self.write(concept=[{'code': 'A00', 'display': 'Choléra', 'designation': [designation]}])
+        self.assertEqual(response.status_code, 201, response.data)
+
+        self.assertEqual(
+            self.read()['concept'][0], {'code': 'A00', 'display': 'Choléra', 'designation': [designation]})
+
+    def test_a_supplement_concept_without_a_name_is_refused_not_dropped(self):
+        """OCL keeps a concept only with a name, and a supplement's write makes none up: the whole write is a 400."""
+        response = self.write(concept=[
+            {'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]},
+            {'code': 'A01', 'property': [{'code': 'inactive', 'valueBoolean': False}]}])
+
+        self.assertEqual(response.status_code, 400)
+        self.assertIn('designation', str(response.data))
+        self.assertFalse(Source.objects.filter(mnemonic='icd10-fr').exists())
+
+    def test_a_supplement_must_name_what_it_supplements(self):
+        response = self.write(supplements=None)
+
+        self.assertEqual(response.status_code, 400)
+        self.assertIn('supplements', str(response.data))
+        self.assertFalse(Source.objects.filter(mnemonic='icd10-fr').exists())
+
+    def test_an_update_carries_supplements_and_its_extensions(self):
+        self.assertEqual(self.write(extension=LANG_PACK).status_code, 201)
+
+        response = self.write(
+            'put', version='2', supplements='http://hl7.org/fhir/sid/icd-10', extension=LANG_PACK[:1],
+            concept=[{'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]}])
+        self.assertEqual(response.status_code, 200, response.data)
+
+        for version in ('HEAD', '2'):
+            source = Source.objects.get(mnemonic='icd10-fr', version=version, user=self.user)
+            self.assertEqual(
+                [source.supplements, source.supplement_type, source.supplement_global],
+                ['http://hl7.org/fhir/sid/icd-10', 'lang-pack', None], version)
+        self.assertEqual(self.names('A00'), [('fr', 'Choléra', False)])
+        resource = self.read()
+        self.assertEqual(
+            [resource['supplements'], resource['extension']], ['http://hl7.org/fhir/sid/icd-10', LANG_PACK[:1]])
+
+    @override_settings(FHIR_CODE_SYSTEM_CONCEPT_LIMIT=1)
+    def test_a_supplement_over_the_concept_limit_is_still_a_supplement(self):
+        """Its concepts are left out (SUBSETTED), but it can't be not-present: it would no longer be a supplement."""
+        response = self.write(concept=[
+            {'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]},
+            {'code': 'A01', 'designation': [{'language': 'fr', 'value': 'Fièvres typhoïde et paratyphoïde'}]}])
+        self.assertEqual(response.status_code, 201, response.data)
+
+        resource = self.read()
+        self.assertEqual([resource['content'], resource['count']], ['supplement', 2])
+        self.assertNotIn('concept', resource)
+        self.assertIn(SUBSETTED, resource['meta']['tag'])
+
+    def test_a_concept_with_designations_but_no_display_loads(self):
+        response = self.write(content='complete', supplements=None, concept=[
+            {'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]}])
+
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(self.names('A00'), [('fr', 'Choléra', False)])
+        concept = self.read()['concept'][0]
+        self.assertEqual([prop['code'] for prop in concept['property']], ['conceptclass', 'datatype'])

core/fhir/adapters/r4.py

@@ -51,8 +51,8 @@ def part_parameter(name, parts):
 SEARCH_PARAMETER_TYPES = {
     '_id': 'token', '_lastUpdated': 'date', 'url': 'uri', 'version': 'token', 'status': 'token', 'identifier': 'token',
     'name': 'string', 'title': 'string', 'description': 'string', 'publisher': 'string', 'system': 'uri',
-    'code': 'token', 'language': 'token', 'reference': 'uri', 'source-system': 'uri', 'source-uri': 'reference',
-    'target-system': 'uri', 'target-uri': 'reference'}
+    'code': 'token', 'language': 'token', 'supplements': 'reference', 'content-mode': 'token', 'reference': 'uri',
+    'source-system': 'uri', 'source-uri': 'reference', 'target-system': 'uri', 'target-uri': 'reference'}
 # R4 defines these once for every resource, or for every conformance resource; the others per resource type
 SHARED_SEARCH_PARAMETERS = {
     '_id': 'Resource-id', '_lastUpdated': 'Resource-lastUpdated',