diff --git a/core/code_systems/constants.py b/core/code_systems/constants.py index 19016cff3..dcedcb7d4 100644 --- a/core/code_systems/constants.py +++ b/core/code_systems/constants.py @@ -1 +1,5 @@ RESOURCE_TYPE = 'CodeSystem' +SUPPLEMENT_TYPE_EXTENSION = 'http://hl7.org/fhir/StructureDefinition/codesystem-supplement-type' +SUPPLEMENT_GLOBAL_EXTENSION = 'http://hl7.org/fhir/StructureDefinition/codesystem-supplement-global' +SUPPLEMENT_CONCEPT_NEEDS_A_NAME = 'A supplement concept needs a display or a designation: OCL keeps a concept only ' \ + 'with a name.' diff --git a/core/code_systems/serializers.py b/core/code_systems/serializers.py index 8b46cff1e..440323815 100644 --- a/core/code_systems/serializers.py +++ b/core/code_systems/serializers.py @@ -2,12 +2,14 @@ import logging from collections import OrderedDict +from pydash import get from rest_framework import serializers from rest_framework.fields import CharField, BooleanField, SerializerMethodField, ChoiceField, \ DateTimeField from core import settings -from core.code_systems.constants import RESOURCE_TYPE +from core.code_systems.constants import RESOURCE_TYPE, SUPPLEMENT_TYPE_EXTENSION, SUPPLEMENT_GLOBAL_EXTENSION, \ + SUPPLEMENT_CONCEPT_NEEDS_A_NAME from core.common.constants import HEAD from core.common.fhir_helpers import delete_empty_fields from core.common.serializers import ReadSerializerMixin, StatusField, IdentifierSerializer @@ -15,6 +17,7 @@ from core.concepts.serializers import ConceptDetailSerializer from core.fhir import search as fhir_search from core.fhir.serializers import CanonicalUrlField, FhirRepositorySerializerMixin, TitleField +from core.sources.constants import CONTENT_TYPE_SUPPLEMENT from core.sources.models import Source from core.sources.serializers import SourceCreateOrUpdateSerializer @@ -111,18 +114,47 @@ def to_internal_value(self, data): if 'names' not in ret: ret.update({'names': []}) - found = False - for concept_name in ret['names']: - if concept_name['name'] == data['display'] and concept_name['locale'] == settings.DEFAULT_LOCALE: + display = data.get('display') + if self.context.get('supplement'): # nothing made up: a display is the name in the code system's language + self.add_preferred_name(ret['names'], display, self.context['language']) + if not ret['names']: # refused here, before anything is saved: the concept wouldn't be + raise serializers.ValidationError({'designation': [SUPPLEMENT_CONCEPT_NEEDS_A_NAME]}) + elif display or not ret['names']: # a concept needs a name: the code, when it has no other + self.add_preferred_name(ret['names'], display or ret['mnemonic'], settings.DEFAULT_LOCALE) + + return ret + + @staticmethod + def add_preferred_name(names, name, locale): + """`name` as the preferred name in `locale`: the designation with that value there, else a new name.""" + if not name: + return + for concept_name in names: + if concept_name['name'] == name and concept_name['locale'] == locale: concept_name['locale_preferred'] = True - found = True - break + return + names.append({'name': name, 'locale': locale, 'locale_preferred': True}) - if not found: - ret['names'].append({'name': data.get('display', data.get('code', None)), 'locale': settings.DEFAULT_LOCALE, - 'locale_preferred': True}) - return ret +class CodeSystemSupplementConceptSerializer(CodeSystemConceptSerializer): + """ + A supplement's concept has only what the supplement adds to the code: its names, and inactive when retired (no + concept class or datatype, which are the supplemented code system's). Its preferred name in the code system's + language (context language) is its display, FHIR's primary designation, and is repeated as a designation only + when it has a use of its own. + """ + def to_representation(self, instance): + names = list(instance.names.all()) + display = next((name for name in names if name.locale_preferred and not name.retired and + name.locale == self.context['language']), None) + return OrderedDict([ + ('code', instance.mnemonic), + ('display', display.name if display else None), + ('definition', self.fields['definition'].to_representation(instance)), + ('designation', CodeSystemConceptDesignationSerializer( + [name for name in names if name is not display or name.type], many=True).data), + ('property', [{'code': 'inactive', 'valueBoolean': True}] if instance.retired else []), + ]) class CodeSystemPropertySerializer(ReadSerializerMixin, serializers.Serializer): @@ -143,14 +175,48 @@ def listed_concepts(context, source): class CodeSystemConceptField(serializers.Field): def to_internal_value(self, data): - concepts = CodeSystemConceptSerializer(data=data, many=True) + """The concepts written, as the code system they're written to says: its content and its language.""" + code_system, instance = self.parent.initial_data, self.parent.instance + content = code_system.get('content', get(instance, 'content_type')) + language = code_system.get('language') or get(instance, 'default_locale') or settings.DEFAULT_LOCALE + concepts = CodeSystemConceptSerializer( + data=data, many=True, context={'supplement': content == CONTENT_TYPE_SUPPLEMENT, 'language': language}) concepts.is_valid(raise_exception=True) return {'concepts': concepts.validated_data} def to_representation(self, value): - """Every concept, retired ones included, up to the limit; none above it (content not-present).""" + """ + Every concept, retired ones included, up to the limit; none above it: content not-present, or for a supplement, + which stays one, SUBSETTED. + """ concepts = listed_concepts(self.context, value)[1] - return CodeSystemConceptSerializer(concepts, many=True).data if concepts else [] + self.subsetted = concepts is None and value.is_supplement + if not concepts: + return [] + if value.is_supplement: + return CodeSystemSupplementConceptSerializer( + concepts, many=True, context={'language': value.default_locale}).data + return CodeSystemConceptSerializer(concepts, many=True).data + + +class CodeSystemExtensionField(serializers.Field): + """The extensions OCL keeps: a supplement's codesystem-supplement-type and codesystem-supplement-global.""" + def to_internal_value(self, data): + if not isinstance(data, list): + raise serializers.ValidationError('extension must be a list.') + extensions = {item.get('url'): item for item in data if isinstance(item, dict)} + return { + 'supplement_type': extensions.get(SUPPLEMENT_TYPE_EXTENSION, {}).get('valueCode'), + 'supplement_global': extensions.get(SUPPLEMENT_GLOBAL_EXTENSION, {}).get('valueBoolean'), + } + + def to_representation(self, value): + extensions = [] + if value.supplement_type: + extensions.append({'url': SUPPLEMENT_TYPE_EXTENSION, 'valueCode': value.supplement_type}) + if value.supplement_global is not None: + extensions.append({'url': SUPPLEMENT_GLOBAL_EXTENSION, 'valueBoolean': value.supplement_global}) + return extensions class TextField(ReadSerializerMixin, serializers.Serializer): @@ -187,6 +253,8 @@ class CodeSystemDetailSerializer(FhirRepositorySerializerMixin, serializers.Mode choices=['not-present', 'example', 'fragment', 'complete', 'supplement'], required=True ) + supplements = CharField(required=False) + extension = CodeSystemExtensionField(source='*', required=False) property = SerializerMethodField() meta = SerializerMethodField() concept = CodeSystemConceptField(source='*', required=False) @@ -201,7 +269,8 @@ class CodeSystemDetailSerializer(FhirRepositorySerializerMixin, serializers.Mode class Meta: model = Source - fields = ('resourceType', 'url', 'title', 'status', 'id', 'language', 'count', 'content', 'property', 'meta', + fields = ('resourceType', 'extension', 'url', 'title', 'status', 'id', 'language', 'count', 'content', + 'supplements', 'property', 'meta', 'version', 'identifier', 'contact', 'jurisdiction', 'name', 'description', 'publisher', 'purpose', 'copyright', 'revisionDate', 'experimental', 'caseSensitive', 'compositional', 'versionNeeded', 'collectionReference', 'hierarchyMeaning', 'concept', 'text', 'filters') @@ -217,8 +286,10 @@ def get_total_count(self, obj): def get_content(self, obj): """ complete (or the content mode the CodeSystem was saved with) up to the concept limit, else not-present: a - search annotates concepts_over_limit, a read counts them. + search annotates concepts_over_limit, a read counts them. A supplement is one at any size. """ + if obj.is_supplement: + return CONTENT_TYPE_SUPPLEMENT over_limit = getattr(obj, 'concepts_over_limit', None) if over_limit is None: over_limit = listed_concepts(self.context, obj)[1] is None @@ -248,6 +319,8 @@ def get_property(obj): 'type': 'Coding' } ] + if obj.is_supplement: # its concepts have no concept class or datatype (CodeSystemSupplementConceptSerializer) + properties = [prop for prop in properties if prop['code'] == 'inactive'] new_properties = [*(obj.properties or [])] defined_property_codes = {prop['code'] for prop in new_properties} for prop in properties: diff --git a/core/code_systems/tests.py b/core/code_systems/tests.py index 2dc64df63..1e0c1fa7c 100644 --- a/core/code_systems/tests.py +++ b/core/code_systems/tests.py @@ -1,5 +1,6 @@ import json +from django.test import override_settings from fhir.resources.R4B.codesystem import CodeSystem from mock.mock import patch, Mock from rest_framework.test import APIClient @@ -10,6 +11,7 @@ from core.common.tests import OCLTestCase from core.concepts.models import Concept from core.concepts.tests.factories import ConceptFactory +from core.fhir.serializers import SUBSETTED from core.orgs.tests.factories import OrganizationFactory from core.sources.models import Source from core.sources.tests.factories import OrganizationSourceFactory, UserSourceFactory @@ -764,3 +766,143 @@ def test_conversion_to_fhir(self): self.assertTrue(can_convert) code_system = CodeSystemConverter.to_fhir(source) self.assertEqual(code_system['id'], 'test') + + +SUPPLEMENTED = 'http://hl7.org/fhir/sid/icd-10|2019-covid-expanded' +LANG_PACK = [ + {'url': 'http://hl7.org/fhir/StructureDefinition/codesystem-supplement-type', 'valueCode': 'lang-pack'}, + {'url': 'http://hl7.org/fhir/StructureDefinition/codesystem-supplement-global', 'valueBoolean': True}] + + +@patch('core.common.tasks.__handle_save', Mock(__name__='__handle_save')) +@patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) +@patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) +class CodeSystemSupplementTest(OCLTestCase): + """ + A supplement written and read as FHIR shows one, like HL7 Denmark's ICD-10 supplement on tx.fhir.org: content + supplement, supplements, and concepts that are a code and their designations (OpenConceptLab/ocl_issues#3030). + """ + def setUp(self): + super().setUp() + self.user = UserProfileFactory() + self.client = APIClient(HTTP_AUTHORIZATION='Token ' + self.user.get_token()) + self.path = f'/users/{self.user.mnemonic}/CodeSystem/' + + def write(self, method='post', **resource): + body = { + 'resourceType': 'CodeSystem', 'id': 'icd10-fr', 'url': 'http://example.org/CodeSystem/icd10-fr', + 'version': '1', 'name': 'icd10-fr', 'title': 'ICD-10 fr', 'status': 'active', 'language': 'fr', + 'content': 'supplement', 'supplements': SUPPLEMENTED, **resource} + path = self.path + ('icd10-fr/' if method == 'put' else '') + return getattr(self.client, method)( + path, {key: value for key, value in body.items() if value is not None}, format='json') + + def read(self): + response = self.client.get(self.path + 'icd10-fr/') + self.assertEqual(response.status_code, 200, response.data) + return response.data + + def head(self): + return Source.objects.get(mnemonic='icd10-fr', version='HEAD', user=self.user) + + def names(self, code): + concept = self.head().get_concepts_queryset().get(mnemonic=code) + return sorted((name.locale, name.name, name.locale_preferred) for name in concept.names.all()) + + def test_a_supplement_is_written_and_read_without_anything_added(self): + response = self.write(extension=LANG_PACK, concept=[ + {'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]}, + {'code': 'A01', 'designation': [{'language': 'fr', 'value': 'Fièvres typhoïde et paratyphoïde'}]}]) + self.assertEqual(response.status_code, 201, response.data) + + head = self.head() + self.assertEqual( + [head.content_type, head.supplements, head.supplement_type, head.supplement_global], + ['supplement', SUPPLEMENTED, 'lang-pack', True]) + self.assertEqual(self.names('A00'), [('fr', 'Choléra', False)]) # no name made up from the code + + resource = self.read() + self.assertEqual( + [resource['content'], resource['supplements'], resource['extension']], + ['supplement', SUPPLEMENTED, LANG_PACK]) + self.assertEqual( + resource['concept'][0], {'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]}) + self.assertEqual([prop['code'] for prop in resource['property']], ['inactive']) + + def test_a_supplements_display_is_its_name_in_the_code_systems_language(self): + """FHIR: the display is a designation in the resource's language with the use 'primary'.""" + response = self.write(concept=[{'code': 'A00', 'display': 'Choléra', 'designation': [ + {'language': 'fr', 'value': 'Choléra, sans précision'}, {'language': 'en', 'value': 'Cholera'}]}]) + self.assertEqual(response.status_code, 201, response.data) + + self.assertEqual( + self.names('A00'), + [('en', 'Cholera', False), ('fr', 'Choléra', True), ('fr', 'Choléra, sans précision', False)]) + self.assertEqual(self.read()['concept'][0], { + 'code': 'A00', 'display': 'Choléra', 'designation': [ + {'language': 'fr', 'value': 'Choléra, sans précision'}, {'language': 'en', 'value': 'Cholera'}]}) + + def test_a_designation_with_a_use_is_kept_when_it_is_also_the_display(self): + designation = {'language': 'fr', 'value': 'Choléra', 'use': {'code': 'synonym'}} + response = self.write(concept=[{'code': 'A00', 'display': 'Choléra', 'designation': [designation]}]) + self.assertEqual(response.status_code, 201, response.data) + + self.assertEqual( + self.read()['concept'][0], {'code': 'A00', 'display': 'Choléra', 'designation': [designation]}) + + def test_a_supplement_concept_without_a_name_is_refused_not_dropped(self): + """OCL keeps a concept only with a name, and a supplement's write makes none up: the whole write is a 400.""" + response = self.write(concept=[ + {'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]}, + {'code': 'A01', 'property': [{'code': 'inactive', 'valueBoolean': False}]}]) + + self.assertEqual(response.status_code, 400) + self.assertIn('designation', str(response.data)) + self.assertFalse(Source.objects.filter(mnemonic='icd10-fr').exists()) + + def test_a_supplement_must_name_what_it_supplements(self): + response = self.write(supplements=None) + + self.assertEqual(response.status_code, 400) + self.assertIn('supplements', str(response.data)) + self.assertFalse(Source.objects.filter(mnemonic='icd10-fr').exists()) + + def test_an_update_carries_supplements_and_its_extensions(self): + self.assertEqual(self.write(extension=LANG_PACK).status_code, 201) + + response = self.write( + 'put', version='2', supplements='http://hl7.org/fhir/sid/icd-10', extension=LANG_PACK[:1], + concept=[{'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]}]) + self.assertEqual(response.status_code, 200, response.data) + + for version in ('HEAD', '2'): + source = Source.objects.get(mnemonic='icd10-fr', version=version, user=self.user) + self.assertEqual( + [source.supplements, source.supplement_type, source.supplement_global], + ['http://hl7.org/fhir/sid/icd-10', 'lang-pack', None], version) + self.assertEqual(self.names('A00'), [('fr', 'Choléra', False)]) + resource = self.read() + self.assertEqual( + [resource['supplements'], resource['extension']], ['http://hl7.org/fhir/sid/icd-10', LANG_PACK[:1]]) + + @override_settings(FHIR_CODE_SYSTEM_CONCEPT_LIMIT=1) + def test_a_supplement_over_the_concept_limit_is_still_a_supplement(self): + """Its concepts are left out (SUBSETTED), but it can't be not-present: it would no longer be a supplement.""" + response = self.write(concept=[ + {'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]}, + {'code': 'A01', 'designation': [{'language': 'fr', 'value': 'Fièvres typhoïde et paratyphoïde'}]}]) + self.assertEqual(response.status_code, 201, response.data) + + resource = self.read() + self.assertEqual([resource['content'], resource['count']], ['supplement', 2]) + self.assertNotIn('concept', resource) + self.assertIn(SUBSETTED, resource['meta']['tag']) + + def test_a_concept_with_designations_but_no_display_loads(self): + response = self.write(content='complete', supplements=None, concept=[ + {'code': 'A00', 'designation': [{'language': 'fr', 'value': 'Choléra'}]}]) + + self.assertEqual(response.status_code, 201, response.data) + self.assertEqual(self.names('A00'), [('fr', 'Choléra', False)]) + concept = self.read()['concept'][0] + self.assertEqual([prop['code'] for prop in concept['property']], ['conceptclass', 'datatype']) diff --git a/core/fhir/adapters/r4.py b/core/fhir/adapters/r4.py index 02fd4abb2..b60703ee8 100644 --- a/core/fhir/adapters/r4.py +++ b/core/fhir/adapters/r4.py @@ -51,8 +51,8 @@ def part_parameter(name, parts): SEARCH_PARAMETER_TYPES = { '_id': 'token', '_lastUpdated': 'date', 'url': 'uri', 'version': 'token', 'status': 'token', 'identifier': 'token', 'name': 'string', 'title': 'string', 'description': 'string', 'publisher': 'string', 'system': 'uri', - 'code': 'token', 'language': 'token', 'reference': 'uri', 'source-system': 'uri', 'source-uri': 'reference', - 'target-system': 'uri', 'target-uri': 'reference'} + 'code': 'token', 'language': 'token', 'supplements': 'reference', 'content-mode': 'token', 'reference': 'uri', + 'source-system': 'uri', 'source-uri': 'reference', 'target-system': 'uri', 'target-uri': 'reference'} # R4 defines these once for every resource, or for every conformance resource; the others per resource type SHARED_SEARCH_PARAMETERS = { '_id': 'Resource-id', '_lastUpdated': 'Resource-lastUpdated', diff --git a/core/fhir/search.py b/core/fhir/search.py index fffce1964..b112a3f99 100644 --- a/core/fhir/search.py +++ b/core/fhir/search.py @@ -21,7 +21,8 @@ parameter within the same time. - A search entry is the resource's summary, marked SUBSETTED: no concept, compose or group, which only a read returns. A CodeSystem is `complete` (with every concept, retired ones flagged) up to FHIR_CODE_SYSTEM_CONCEPT_LIMIT concepts, - else `not-present` (design D9). + else `not-present` (design D9). A supplement is `supplement` at any size; past the limit its concepts are left out + and the read is marked SUBSETTED. - A version released before its repository had a canonical URL has the repository's (HEAD's), in its `url` and for the url parameter. """ @@ -58,6 +59,7 @@ from core.fhir.utils import fhir_base_url from core.mappings.models import Mapping from core.orgs.models import Organization +from core.sources.constants import CONTENT_TYPE_SUPPLEMENT from core.sources.models import Source from core.users.models import UserProfile @@ -82,7 +84,9 @@ '_id': 'by_id', '_lastUpdated': 'by_last_updated', 'url': 'by_url', 'version': 'by_version', 'status': 'by_status', 'identifier': 'by_identifier'} PARAMETERS = { - 'CodeSystem': {**COMMON_PARAMETERS, 'system': 'by_url', 'code': 'by_code', 'language': 'by_language'}, + 'CodeSystem': { + **COMMON_PARAMETERS, 'system': 'by_url', 'code': 'by_code', 'language': 'by_language', + 'supplements': 'by_supplements', 'content-mode': 'by_content_mode'}, 'ValueSet': {**COMMON_PARAMETERS, 'reference': 'by_reference'}, 'ConceptMap': { **COMMON_PARAMETERS, 'source-system': 'by_source', 'source-uri': 'by_source', 'target-system': 'by_target', @@ -99,9 +103,12 @@ CODE_SYSTEM = 'the code system searched' TOKEN_SYSTEMS = { '_id': None, 'version': None, 'status': 'http://hl7.org/fhir/publication-status', 'language': 'urn:ietf:bcp:47', - 'code': CODE_SYSTEM} + 'code': CODE_SYSTEM, 'content-mode': 'http://hl7.org/fhir/codesystem-content-mode'} # what system| (any code of the system) means for a parameter with a system of its own -ANY_CODE = {'status': EVERYTHING, 'language': Q(default_locale__isnull=False) & ~Q(default_locale='')} +ANY_CODE = { + 'status': EVERYTHING, 'language': Q(default_locale__isnull=False) & ~Q(default_locale=''), + 'content-mode': EVERYTHING} +STATED_CONTENT = ('example', 'fragment') # the content modes a CodeSystem shows as stored, up to the concept limit STATUSES = {'active': Q(released=True, retired=False), 'retired': Q(retired=True)} # search shows no draft # the parameters whose values are canonical URLs, looked up before the search (Search.look_up), by what they're among LOOKUPS = { @@ -233,11 +240,10 @@ def code_system_concepts(source): return count, concepts -def with_concepts_over_limit(queryset): - """Each Source row annotated with concepts_over_limit, reading at most one link past the limit.""" +def concepts_over_limit(): + """Whether a Source row has more concepts than a read lists, reading at most one link past the limit.""" limit = settings.FHIR_CODE_SYSTEM_CONCEPT_LIMIT - links = Concept.sources.through.objects.filter(source_id=OuterRef('id')) - return queryset.annotate(concepts_over_limit=Exists(links[limit:limit + 1])) + return Exists(Concept.sources.through.objects.filter(source_id=OuterRef('id'))[limit:limit + 1]) def defines_concept(**filters): @@ -517,6 +523,28 @@ def by_code(value): def by_language(value): return Q(default_locale=value) + @staticmethod + def by_supplements(value): + """A CodeSystem supplementing the canonical URL named: any version of it, or only the one url|version names.""" + url, _, version = value.partition('|') + if version: + return Q(supplements=value) + return Q(supplements=url) | Q(supplements__startswith=f'{url}|') + + @staticmethod + def by_content_mode(value): + """The content a CodeSystem shows (CodeSystemDetailSerializer.get_content): a supplement's at any size.""" + supplement, over_limit = Q(content_type=CONTENT_TYPE_SUPPLEMENT), concepts_over_limit() + if value == CONTENT_TYPE_SUPPLEMENT: + return supplement + if value == 'not-present': + return ~supplement & over_limit + if value in STATED_CONTENT: + return Q(content_type=value) & ~over_limit + if value == 'complete': + return ~Q(content_type__in=[CONTENT_TYPE_SUPPLEMENT, *STATED_CONTENT]) & ~over_limit + return NOTHING + def by_reference(self, value): """ A ValueSet whose stored default expansion draws on a version of the code system (canonical URL) named, among @@ -585,7 +613,7 @@ def search_bundle(view): rows = with_repository_canonical(REPO_TYPES[resource_type].objects.filter( id__in=ids[(page - 1) * count:page * count])).order_by('-id') if resource_type == 'CodeSystem': - rows = with_concepts_over_limit(rows) + rows = rows.annotate(concepts_over_limit=concepts_over_limit()) paginator = CustomPaginator( request=request, total_count=len(ids), queryset=list(rows), page_size=count, is_sliced=True) resources = view.get_serializer(paginator.current_page_results, many=True).data diff --git a/core/fhir/swagger.py b/core/fhir/swagger.py index cfdc1738f..35c766f55 100644 --- a/core/fhir/swagger.py +++ b/core/fhir/swagger.py @@ -27,6 +27,8 @@ 'system': 'Canonical URL of the code system (an alias of url).', 'code': 'A code defined by the code system; code or system|code.', 'language': 'Language token for the code system (urn:ietf:bcp:47).', + 'supplements': 'Canonical URL of the code system a supplement supplements: any version of it, or url|version.', + 'content-mode': 'Content token: not-present, example, fragment, complete or supplement, as the read shows it.', 'reference': 'Canonical code system URL referenced by the value set\'s stored default expansion.', 'source-system': 'Canonical URL of a source system in a ConceptMap group.', 'source-uri': 'Alias of source-system.', diff --git a/core/fhir/tests_search.py b/core/fhir/tests_search.py index 86b8133a2..7e4da3e2e 100644 --- a/core/fhir/tests_search.py +++ b/core/fhir/tests_search.py @@ -589,6 +589,56 @@ def test_releases_without_a_canonical_url_of_their_own_add_no_query(self): self.assertLessEqual(unnamed, set(self.keys(bundle))) +@tag('es') +class CodeSystemSupplementSearchTest(SearchCorpusMixin, OCLTestCase): + """The supplements and content-mode parameters (OpenConceptLab/ocl_issues#3030).""" + FR = 'https://who.example/CodeSystem/icd10-fr' + PT = 'https://who.example/CodeSystem/icd10-pt' + + @classmethod + def setUpTestData(cls): + super().setUpTestData() + fr = {'canonical_url': cls.FR, 'supplements': f'{ICD10}|2019'} + pt = {'canonical_url': cls.PT, 'supplements': ICD10} + cls.fr = source(cls.who, 'icd10-fr', content_type='supplement', **fr) + cls.fr_v1 = source(cls.who, 'icd10-fr', 'v1', released=True, content_type='supplement', **fr) + cls.pt = source(cls.who, 'icd10-pt', content_type='supplement', **pt) + cls.pt_v1 = source(cls.who, 'icd10-pt', 'v1', released=True, content_type='supplement', **pt) + + def test_supplements_matches_any_version_unless_one_is_named(self): + self.assertEqual(self.keys(self.search('CodeSystem', supplements=ICD10)), [f'{self.FR}|v1', f'{self.PT}|v1']) + self.assertEqual(self.keys(self.search('CodeSystem', supplements=f'{ICD10}|2019')), [f'{self.FR}|v1']) + self.assertEqual(self.search('CodeSystem', supplements=f'{ICD10}|2020')['total'], 0) + self.assertEqual(self.search('CodeSystem', supplements='http://hl7.org/fhir/sid/icd')['total'], 0) + self.assertEqual(self.search('CodeSystem', base='/orgs/CIEL', supplements=ICD10)['total'], 0) + + def test_content_mode_is_the_content_a_read_shows(self): + """A supplement stays one past the concept limit; any other code system is then not-present.""" + mode = 'content-mode' + supplements = [f'{self.FR}|v1', f'{self.PT}|v1'] + ConceptFactory(parent=self.ciel_source, sources=[self.ciel_v2]) + ConceptFactory(parent=self.fr, sources=[self.fr_v1]) + + self.assertEqual(self.keys(self.search('CodeSystem', **{mode: 'supplement'})), supplements) + self.assertEqual( + self.keys(self.search('CodeSystem', **{mode: 'complete'})), CodeSystemSearchTest.PUBLIC_RELEASES) + self.assertEqual(self.search('CodeSystem', **{mode: 'not-present'})['total'], 0) + self.assertEqual( + self.keys(self.search('CodeSystem', **{mode: 'http://hl7.org/fhir/codesystem-content-mode|supplement'})), + supplements) + self.assertEqual(self.search('CodeSystem', **{mode: 'urn:x|supplement'})['total'], 0) + Source.objects.filter(id=self.jdoe_codes_v1.id).update(content_type='example') + self.assertEqual(self.keys(self.search('CodeSystem', **{mode: 'example'})), ['https://example.org/my-codes|v1']) + + with override_settings(FHIR_CODE_SYSTEM_CONCEPT_LIMIT=0): + self.assertEqual(self.keys(self.search('CodeSystem', **{mode: 'not-present'})), [f'{CIEL}|v2']) + self.assertEqual(self.keys(self.search('CodeSystem', **{mode: 'supplement'})), supplements) + self.assertNotIn(f'{CIEL}|v2', self.keys(self.search('CodeSystem', **{mode: 'complete'}))) + self.assertEqual( + {entry['resource']['version']: entry['resource']['content'] + for entry in self.search('CodeSystem', url=self.FR)['entry']}, {'v1': 'supplement'}) + + @tag('es') class ValueSetSearchTest(SearchCorpusMixin, OCLTestCase): def test_global_search_and_the_repository_canonical(self): diff --git a/core/importers/models.py b/core/importers/models.py index 64c06631e..d999b4728 100644 --- a/core/importers/models.py +++ b/core/importers/models.py @@ -253,8 +253,9 @@ class SourceImporter(BaseResourceImporter): "id", "short_code", "name", "full_name", "description", "source_type", "custom_validation_schema", "public_access", "default_locale", "supported_locales", "website", "extras", "external_id", 'canonical_url', 'identifier', 'contact', 'jurisdiction', 'publisher', 'purpose', 'copyright', - 'revision_date', 'text', 'content_type', 'experimental', 'case_sensitive', 'collection_reference', - 'hierarchy_meaning', 'compositional', 'version_needed', 'meta', 'properties', 'filters' + 'revision_date', 'text', 'content_type', 'supplements', 'supplement_type', 'supplement_global', + 'experimental', 'case_sensitive', 'collection_reference', 'hierarchy_meaning', 'compositional', + 'version_needed', 'meta', 'properties', 'filters' ] @staticmethod diff --git a/core/importers/tests.py b/core/importers/tests.py index 28b070362..a3f6241ae 100644 --- a/core/importers/tests.py +++ b/core/importers/tests.py @@ -2209,6 +2209,24 @@ def test_importers_store_public_edit_as_view(self): self.assertEqual(Organization.objects.get(mnemonic='EditOrg').public_access, 'View') self.assertEqual(Source.objects.get(mnemonic='edit-src').public_access, 'View') + def test_source_importer_creates_a_supplement(self): + admin = UserProfile.objects.get(username='ocladmin') + OrganizationFactory(mnemonic='SuppOrg') + supplement = { + 'id': 'icd10-fr', 'name': 'ICD-10 fr', 'owner_type': 'Organization', 'owner': 'SuppOrg', + 'content_type': 'supplement', 'supplements': 'http://hl7.org/fhir/sid/icd-10|2019', + 'supplement_type': 'lang-pack', 'supplement_global': False} + + self.assertEqual(SourceImporter(supplement, admin).run(), CREATED) + source = Source.objects.get(mnemonic='icd10-fr') + self.assertEqual( + [source.supplements, source.supplement_type, source.supplement_global], + ['http://hl7.org/fhir/sid/icd-10|2019', 'lang-pack', False]) + + result = SourceImporter({**supplement, 'id': 'icd10-pt', 'supplements': None}, admin).run() + self.assertIn('supplements', result) + self.assertFalse(Source.objects.filter(mnemonic='icd10-pt').exists()) + def test_source_importer_delete_not_found(self): importer = SourceImporter( {'id': 'no-such-source', 'owner_type': 'Organization', 'owner': 'NoSuchOrg'}, UserProfileFactory() diff --git a/core/integration_tests/tests_sources.py b/core/integration_tests/tests_sources.py index 891fd5f95..bab9761f2 100644 --- a/core/integration_tests/tests_sources.py +++ b/core/integration_tests/tests_sources.py @@ -196,7 +196,8 @@ def test_post_201(self): 'url', 'owner', 'owner_type', 'owner_url', 'created_on', 'updated_on', 'created_by', 'updated_by', 'extras', 'external_id', 'versions_url', 'version', 'concepts_url', 'mappings_url', 'canonical_url', 'identifier', 'publisher', 'contact', 'meta', - 'jurisdiction', 'purpose', 'copyright', 'content_type', 'revision_date', 'logo_url', 'text', + 'jurisdiction', 'purpose', 'copyright', 'content_type', 'supplements', 'supplement_type', + 'supplement_global', 'revision_date', 'logo_url', 'text', 'experimental', 'case_sensitive', 'collection_reference', 'hierarchy_meaning', 'compositional', 'version_needed', 'hierarchy_root_url', 'autoid_concept_mnemonic', 'autoid_mapping_mnemonic', 'autoid_concept_external_id', 'autoid_mapping_external_id', @@ -318,7 +319,8 @@ def test_put_200(self): 'url', 'owner', 'owner_type', 'owner_url', 'created_on', 'updated_on', 'created_by', 'updated_by', 'extras', 'external_id', 'versions_url', 'version', 'concepts_url', 'mappings_url', 'canonical_url', 'identifier', 'publisher', 'contact', 'meta', - 'jurisdiction', 'purpose', 'copyright', 'content_type', 'revision_date', 'logo_url', 'text', + 'jurisdiction', 'purpose', 'copyright', 'content_type', 'supplements', 'supplement_type', + 'supplement_global', 'revision_date', 'logo_url', 'text', 'experimental', 'case_sensitive', 'collection_reference', 'hierarchy_meaning', 'compositional', 'version_needed', 'hierarchy_root_url', 'autoid_concept_mnemonic', 'autoid_mapping_mnemonic', 'autoid_concept_external_id', 'autoid_mapping_external_id', diff --git a/core/sources/constants.py b/core/sources/constants.py index 6702e5300..a227c19b6 100644 --- a/core/sources/constants.py +++ b/core/sources/constants.py @@ -5,6 +5,9 @@ DELETE_SUCCESS = 'Successfully deleted Source.' VERSION_ALREADY_EXISTS = "Source version '{}' already exist." HIERARCHY_ROOT_MUST_BELONG_TO_SAME_SOURCE = 'Hierarchy Root must belong to the same Source.' +CONTENT_TYPE_SUPPLEMENT = 'supplement' +SUPPLEMENTS_REQUIRED = 'A supplement must name the code system it supplements.' +SUPPLEMENTS_FORMAT = 'Enter the canonical URL of the code system, optionally followed by |version.' HIERARCHY_MEANINGS = ( ('grouped-by', 'grouped-by'), ('is-a', 'is-a'), diff --git a/core/sources/migrations/0047_source_supplements.py b/core/sources/migrations/0047_source_supplements.py new file mode 100644 index 000000000..bf7032ec0 --- /dev/null +++ b/core/sources/migrations/0047_source_supplements.py @@ -0,0 +1,29 @@ +# Generated by Django 5.2.17 on 2026-11-13 22:08 + +import core.common.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sources', '0046_retire_public_edit_access'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='supplement_global', + field=models.BooleanField(blank=True, default=None, null=True), + ), + migrations.AddField( + model_name='source', + name='supplement_type', + field=models.CharField(blank=True, max_length=50, null=True), + ), + migrations.AddField( + model_name='source', + name='supplements', + field=core.common.fields.URIField(blank=True, max_length=300, null=True), + ), + ] diff --git a/core/sources/models.py b/core/sources/models.py index e254b8f4b..421952ff5 100644 --- a/core/sources/models.py +++ b/core/sources/models.py @@ -13,6 +13,7 @@ from core.common.checksums import VersionCompareMixin from core.common.constants import HEAD +from core.common.fields import URIField from core.common.models import ConceptContainerModel from core.common.tasks import update_mappings_source, index_source_concepts, index_source_mappings, \ sync_source_concept_vectors, index_source_public_access @@ -21,7 +22,8 @@ from core.concepts.models import ConceptName, Concept from core.services.storages.postgres import PostgresQL from core.sources.constants import SOURCE_TYPE, SOURCE_VERSION_TYPE, HIERARCHY_ROOT_MUST_BELONG_TO_SAME_SOURCE, \ - HIERARCHY_MEANINGS, AUTO_ID_CHOICES, AUTO_ID_SEQUENTIAL, AUTO_ID_UUID, LOCALE_EXTERNAL_AUTO_ID_CHOICES + HIERARCHY_MEANINGS, AUTO_ID_CHOICES, AUTO_ID_SEQUENTIAL, AUTO_ID_UUID, LOCALE_EXTERNAL_AUTO_ID_CHOICES, \ + CONTENT_TYPE_SUPPLEMENT, SUPPLEMENTS_REQUIRED, SUPPLEMENTS_FORMAT from core.tasks.models import Task @@ -101,6 +103,11 @@ class Meta: ('example', 'example'), ('fragment', 'fragment'), ('complete', 'complete'), ('supplement', 'supplement')]) + # A supplement's code system: its canonical URL, optionally |version. The type and global flag are the publisher's + # FHIR codesystem-supplement-type and codesystem-supplement-global extensions, kept as they're given. + supplements = URIField(null=True, blank=True, max_length=300) + supplement_type = models.CharField(null=True, blank=True, max_length=50) + supplement_global = models.BooleanField(null=True, blank=True, default=None) collection_reference = models.CharField(null=True, blank=True, max_length=100) hierarchy_meaning = models.CharField(null=True, blank=True, max_length=50, choices=HIERARCHY_MEANINGS) case_sensitive = models.BooleanField(null=True, blank=True, default=None) @@ -222,6 +229,9 @@ def update_version_data(self, head): super().update_version_data(head) self.source_type = head.source_type self.content_type = head.content_type + self.supplements = head.supplements + self.supplement_type = head.supplement_type + self.supplement_global = head.supplement_global self.collection_reference = head.collection_reference self.hierarchy_meaning = head.hierarchy_meaning self.hierarchy_root_id = head.hierarchy_root_id @@ -274,9 +284,14 @@ def is_hierarchy_root_belonging_to_self(self): return hierarchy_root.parent_id == self.id return hierarchy_root.parent_id == self.head.id + @property + def is_supplement(self): + return self.content_type == CONTENT_TYPE_SUPPLEMENT + def clean(self): self.hierarchy_meaning = self.hierarchy_meaning or None + self.clean_supplements() self.clean_properties() self.clean_filters() self.clean_match_algorithms() @@ -285,6 +300,20 @@ def clean(self): if self.hierarchy_root_id and not self.is_hierarchy_root_belonging_to_self(): raise ValidationError({'hierarchy_root': [HIERARCHY_ROOT_MUST_BELONG_TO_SAME_SOURCE]}) + def clean_supplements(self): + """ + A supplement must name what it supplements (FHIR csd-4); any source may. The URL itself is checked as a URI + (clean_fields); here, at most one |version, which isn't empty. + """ + self.supplements = self.supplements or None + if not self.supplements: + if self.is_supplement: + raise ValidationError({'supplements': [SUPPLEMENTS_REQUIRED]}) + return + version = self.supplements.split('|')[1:] + if len(version) > 1 or version == ['']: + raise ValidationError({'supplements': [SUPPLEMENTS_FORMAT]}) + def clean_match_algorithms(self): self.match_algorithms = self.match_algorithms or [] diff --git a/core/sources/serializers.py b/core/sources/serializers.py index 30a9f3eba..46818c6b6 100644 --- a/core/sources/serializers.py +++ b/core/sources/serializers.py @@ -149,6 +149,7 @@ def get_external_exports(obj): class SourceCreateOrUpdateSerializer(ModelSerializer): canonical_url = CharField(allow_blank=True, allow_null=True, required=False) + supplements = CharField(allow_blank=True, allow_null=True, required=False) public_access = PublicAccessField(required=False) class Meta: @@ -179,8 +180,8 @@ def prepare_object(self, validated_data, instance=None): 'retired', 'released', 'organization', 'user', 'organization_id', 'user_id', 'external_id', 'extras', 'experimental', 'case_sensitive', 'collection_reference', 'hierarchy_meaning', 'compositional', 'version_needed', 'canonical_url', 'identifier', 'publisher', 'contact', 'jurisdiction', 'purpose', - 'copyright', 'content_type', 'revision_date', 'text', 'meta', - 'autoid_concept_mnemonic', 'autoid_concept_external_id', + 'copyright', 'content_type', 'supplements', 'supplement_type', 'supplement_global', 'revision_date', + 'text', 'meta', 'autoid_concept_mnemonic', 'autoid_concept_external_id', 'autoid_mapping_mnemonic', 'autoid_mapping_external_id', 'autoid_concept_mnemonic_start_from', 'autoid_concept_external_id_start_from', 'autoid_mapping_mnemonic_start_from', 'autoid_mapping_external_id_start_from', @@ -425,7 +426,8 @@ class Meta: 'url', 'owner', 'owner_type', 'owner_url', 'created_on', 'updated_on', 'created_by', 'updated_by', 'extras', 'external_id', 'versions_url', 'version', 'concepts_url', 'mappings_url', 'canonical_url', 'identifier', 'publisher', 'contact', 'jurisdiction', 'purpose', 'copyright', - 'content_type', 'revision_date', 'logo_url', 'summary', 'text', 'client_configs', + 'content_type', 'supplements', 'supplement_type', 'supplement_global', 'revision_date', 'logo_url', + 'summary', 'text', 'client_configs', 'experimental', 'case_sensitive', 'collection_reference', 'hierarchy_meaning', 'compositional', 'version_needed', 'hierarchy_root_url', 'hierarchy_root', 'meta', 'autoid_concept_mnemonic', 'autoid_concept_external_id', @@ -520,8 +522,8 @@ class Meta: 'created_on', 'updated_on', 'created_by', 'updated_by', 'extras', 'external_id', 'version', 'concepts_url', 'mappings_url', 'is_processing', 'released', 'canonical_url', 'identifier', 'publisher', 'contact', 'jurisdiction', 'purpose', 'copyright', - 'content_type', 'revision_date', 'summary', 'text', 'meta', - 'experimental', 'case_sensitive', 'collection_reference', 'hierarchy_meaning', 'compositional', + 'content_type', 'supplements', 'supplement_type', 'supplement_global', 'revision_date', 'summary', 'text', + 'meta', 'experimental', 'case_sensitive', 'collection_reference', 'hierarchy_meaning', 'compositional', 'version_needed', 'hierarchy_root_url', 'checksums', 'states', 'tasks', 'properties', 'filters', 'match_algorithms', 'external_exports' ) + AbstractRepoResourcesSerializer.Meta.fields diff --git a/core/sources/tests/tests.py b/core/sources/tests/tests.py index 351a06e91..d85fb221a 100644 --- a/core/sources/tests/tests.py +++ b/core/sources/tests/tests.py @@ -3217,3 +3217,73 @@ def test_migration_moves_public_edit_sources_and_their_content_to_view(self): self.assertTrue(Mapping.objects.filter(id=mapping.id).exists()) self.assertEqual(private_source.public_access, ACCESS_TYPE_NONE) self.assertEqual(private_concept.public_access, ACCESS_TYPE_NONE) + + +class SourceSupplementsTest(OCLAPITestCase): + """A supplement names the code system it supplements (OpenConceptLab/ocl_issues#3030).""" + ICD10 = 'http://hl7.org/fhir/sid/icd-10' + ICD10_2019 = 'http://hl7.org/fhir/sid/icd-10|2019-covid-expanded' + + def setUp(self): + super().setUp() + self.org = OrganizationFactory() + self.user = UserProfileFactory() + self.org.members.add(self.user) + self.token = f'Token {self.user.get_token()}' + + def post(self, **data): + return self.client.post( + f'/orgs/{self.org.mnemonic}/sources/', {'name': 'n', **data}, HTTP_AUTHORIZATION=self.token, + format='json') + + def put(self, path, **data): + return self.client.put(path, data, HTTP_AUTHORIZATION=self.token, format='json') + + def supplement_fields(self, path): + data = self.client.get(path).data + return [data['supplements'], data['supplement_type'], data['supplement_global']] + + def test_a_supplement_must_name_what_it_supplements(self): + response = self.post(id='fr', content_type='supplement') + + self.assertEqual(response.status_code, 400) + self.assertIn('supplements', response.data) + self.assertFalse(Source.objects.filter(mnemonic='fr').exists()) + + def test_supplements_is_a_canonical_url_with_an_optional_version(self): + for index, value in enumerate(['not a url', f'{self.ICD10}|', f'{self.ICD10}|1|2', '|2019']): + response = self.post(id=f'fr-{index}', content_type='supplement', supplements=value) + self.assertEqual(response.status_code, 400, value) + self.assertIn('supplements', response.data, value) + + @patch('core.sources.models.Source.index_children', Mock()) + def test_create_update_read_and_versions_carry_supplements(self): + response = self.post( + id='fr', content_type='supplement', supplements=self.ICD10_2019, supplement_type='lang-pack', + supplement_global=False) + self.assertEqual(response.status_code, 201, response.data) + self.assertEqual( + [response.data['supplements'], response.data['supplement_type'], response.data['supplement_global']], + [self.ICD10_2019, 'lang-pack', False]) + url = response.data['url'] + + response = self.put(url, supplements=self.ICD10, supplement_global=True) + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.supplement_fields(url), [self.ICD10, 'lang-pack', True]) + + response = self.client.post( + f'{url}versions/', {'id': 'v1', 'description': 'v1'}, HTTP_AUTHORIZATION=self.token, format='json') + self.assertEqual(response.status_code, 201, response.data) + self.assertEqual(self.supplement_fields(f'{url}v1/'), [self.ICD10, 'lang-pack', True]) + + response = self.put(url, supplements='') # a supplement can't drop what it supplements + self.assertEqual(response.status_code, 400) + self.assertIn('supplements', response.data) + self.assertEqual(Source.objects.get(uri=url).supplements, self.ICD10) + + def test_supplements_is_optional_for_other_content(self): + """FHIR (csd-4) requires supplements only of a supplement.""" + response = self.post(id='plain', supplements=self.ICD10) + + self.assertEqual(response.status_code, 201, response.data) + self.assertEqual(response.data['supplements'], self.ICD10)