From 442f7828d9eb14f9d7eee695a102902a5b133411 Mon Sep 17 00:00:00 2001 From: Lina Date: Mon, 9 Nov 2026 01:04:16 +0800 Subject: [PATCH 1/2] #809469 check the schema id --- src/dataworkbench/datacatalogue.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/dataworkbench/datacatalogue.py b/src/dataworkbench/datacatalogue.py index e6f2176..c5f9683 100644 --- a/src/dataworkbench/datacatalogue.py +++ b/src/dataworkbench/datacatalogue.py @@ -70,7 +70,7 @@ def save( df: DataFrame, dataset_name: str, dataset_description: str, - schema_id: uuid.UUID | None = None, + schema_id: uuid.UUID, tags: dict[str, str] | None = None, ) -> dict[str, Any]: """ @@ -84,7 +84,7 @@ def save( df: Spark DataFrame containing the dataset to be saved dataset_name: Name of the dataset for the catalog dataset_description: Description of the dataset's purpose and contents - schema_id: Optional UUID for the dataset's schema + schema_id: UUID for the dataset's schema tags: Optional metadata tags as key-value pairs Returns: @@ -102,7 +102,7 @@ def save( ... df, ... "Customer Data", ... "Monthly customer demographic data", - ... "550e8400-e29b-41d4-a716-446655440000", + ... uuid.UUID("550e8400-e29b-41d4-a716-446655440000"), ... {"environment": ["test"]} ... ) """ @@ -116,6 +116,9 @@ def save( if not isinstance(dataset_description, str): raise TypeError("dataset_description must be a string") + if not isinstance(schema_id, uuid.UUID) or not schema_id: + raise TypeError("schema_id must be a non-empty UUID") + if tags is not None and not isinstance(tags, dict): raise TypeError("tags must be a dictionary or None") From 54acb38f4bf4dd31ab7166324f4e3a50ab8fda49 Mon Sep 17 00:00:00 2001 From: Lina Date: Mon, 9 Nov 2026 01:32:45 +0800 Subject: [PATCH 2/2] #809469 support either schema id or schema version id --- README.md | 21 ++++++++- src/dataworkbench/datacatalogue.py | 38 ++++++++++++---- src/dataworkbench/gateway.py | 11 ++++- tests/test_datacatalogue.py | 71 +++++++++++++++++++++++++++--- tests/test_gateway.py | 26 +++++++++++ 5 files changed, 149 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3269962..079bb33 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ datacatalogue.save( #### Using an existing schema When you have an existing schema that you want to reuse: ```python +import uuid + from dataworkbench import DataCatalogue df = spark.createDataFrame([("a", 1), ("b", 2), ("c", 3)], ["letter", "number"]) @@ -82,15 +84,30 @@ datacatalogue.save( "Dataset Name", "Description", tags={"environment": ["test"]}, - schema_id="abada0f7-acb4-43cf-8f54-b51abd7ba8b1" # Using an existing schema ID + schema_id=uuid.UUID("abada0f7-acb4-43cf-8f54-b51abd7ba8b1"), ) ``` +#### Using an existing schema version +When a specific schema version is required: +```python +import uuid + +datacatalogue.save( + df, + "Dataset Name", + "Description", + schema_version_id=uuid.UUID("abada0f7-acb4-43cf-8f54-b51abd7ba8b1"), + tags={"environment": ["test"]}, +) +``` + +Provide exactly one of `schema_id` or `schema_version_id`. ## API Reference ### DataCatalogue -- `save(df, name, description, schema_id=None, tags=None)`: Save a Spark DataFrame to the Data Workbench Data Catalogue +- `save(df, name, description, schema_id=None, tags=None, schema_version_id=None)`: Save a Spark DataFrame to the Data Workbench Data Catalogue. Provide exactly one schema identifier. ## License diff --git a/src/dataworkbench/datacatalogue.py b/src/dataworkbench/datacatalogue.py index c5f9683..908efee 100644 --- a/src/dataworkbench/datacatalogue.py +++ b/src/dataworkbench/datacatalogue.py @@ -70,8 +70,9 @@ def save( df: DataFrame, dataset_name: str, dataset_description: str, - schema_id: uuid.UUID, + schema_id: uuid.UUID | None = None, tags: dict[str, str] | None = None, + schema_version_id: uuid.UUID | None = None, ) -> dict[str, Any]: """ Save a dataset to storage and register it with the data catalog. @@ -84,8 +85,11 @@ def save( df: Spark DataFrame containing the dataset to be saved dataset_name: Name of the dataset for the catalog dataset_description: Description of the dataset's purpose and contents - schema_id: UUID for the dataset's schema + schema_id: UUID for the dataset's schema. Mutually exclusive with + schema_version_id. tags: Optional metadata tags as key-value pairs + schema_version_id: UUID for the dataset schema version. Mutually + exclusive with schema_id. Returns: Dict[str, Any]: Response from the Gateway service containing dataset @@ -102,8 +106,10 @@ def save( ... df, ... "Customer Data", ... "Monthly customer demographic data", - ... uuid.UUID("550e8400-e29b-41d4-a716-446655440000"), - ... {"environment": ["test"]} + ... schema_version_id=uuid.UUID( + ... "550e8400-e29b-41d4-a716-446655440000" + ... ), + ... tags={"environment": ["test"]}, ... ) """ # Validate input parameters @@ -116,12 +122,23 @@ def save( if not isinstance(dataset_description, str): raise TypeError("dataset_description must be a string") - if not isinstance(schema_id, uuid.UUID) or not schema_id: - raise TypeError("schema_id must be a non-empty UUID") - if tags is not None and not isinstance(tags, dict): raise TypeError("tags must be a dictionary or None") + if schema_id is None and schema_version_id is None: + raise ValueError("Either schema_id or schema_version_id must be provided") + + if schema_id is not None and schema_version_id is not None: + raise ValueError("schema_id and schema_version_id cannot both be provided") + + if schema_id is not None and not isinstance(schema_id, uuid.UUID): + raise TypeError("schema_id must be a UUID") + + if schema_version_id is not None and not isinstance( + schema_version_id, uuid.UUID + ): + raise TypeError("schema_version_id must be a UUID") + # Generate folder_id folder_id = uuid.uuid4() @@ -134,7 +151,12 @@ def save( try: # Register the dataset with the Gateway API return self.gateway.import_dataset( - dataset_name, dataset_description, schema_id, tags or {}, folder_id + dataset_name, + dataset_description, + schema_id, + tags or {}, + folder_id, + schema_version_id=schema_version_id, ) except Exception as e: self._rollback_write(folder_id) diff --git a/src/dataworkbench/gateway.py b/src/dataworkbench/gateway.py index bc989b6..911766f 100644 --- a/src/dataworkbench/gateway.py +++ b/src/dataworkbench/gateway.py @@ -111,6 +111,7 @@ def import_dataset( schema_id: UUID | None, tags: dict[str, str], folder_id: UUID, + schema_version_id: UUID | None = None, ) -> dict[str, Any]: """ Imports a dataset into the DataWorkbench. @@ -120,9 +121,12 @@ def import_dataset( Args: dataset_name: Name of the dataset to be imported dataset_description: Description of the dataset's purpose and contents - schema_id: UUID identifier for the dataset's schema + schema_id: UUID identifier for the dataset's schema. Mutually + exclusive with schema_version_id. tags: Metadata tags as key-value pairs for categorization and discovery folder_id: UUID identifier for the folder where the dataset will be stored + schema_version_id: UUID identifier for the dataset's schema version. + Mutually exclusive with schema_id. Returns: Dict[str, Any]: JSON response from the API or an error message dictionary @@ -144,9 +148,12 @@ def import_dataset( "datasetFolderId": str(folder_id), "datasetName": dataset_name, "datasetDescription": dataset_description, - "schemaId": str(schema_id) if schema_id else schema_id, "tags": tags, } + if schema_id is not None: + payload["schemaId"] = str(schema_id) + if schema_version_id is not None: + payload["schemaVersionId"] = str(schema_version_id) logger.info(f"Sending request to import dataset: {dataset_name}") diff --git a/tests/test_datacatalogue.py b/tests/test_datacatalogue.py index 2bf7a75..47b0d9c 100644 --- a/tests/test_datacatalogue.py +++ b/tests/test_datacatalogue.py @@ -31,19 +31,27 @@ def storage_handler(): @patch.object(DeltaStorage, "write", return_value="mock_write_success") @patch.object(Gateway, "import_dataset", return_value="mock_datacatalog_success") -def test_save_dataset(mock_write, mock_gateway_import, mock_dependencies): +def test_save_dataset(mock_gateway_import, mock_write, mock_dependencies): """Test saving a dataset without making real API calls""" datacatalogue, _, _ = mock_dependencies + schema_id = uuid.uuid4() + tags = {"environment": "test"} result = datacatalogue.save( - df=MagicMock(spec=DataFrame), - dataset_name="test_dataset", - dataset_description="test description" + MagicMock(spec=DataFrame), + "test_dataset", + "test description", + schema_id, + tags, ) assert result == "mock_datacatalog_success" mock_write.assert_called_once() mock_gateway_import.assert_called_once() + gateway_args = mock_gateway_import.call_args.args + assert gateway_args[:4] == ("test_dataset", "test description", schema_id, tags) + assert isinstance(gateway_args[4], uuid.UUID) + assert mock_gateway_import.call_args.kwargs == {"schema_version_id": None} @pytest.mark.parametrize("folder_id", ["", 123, "5f69754e-37a0-431b-aa3e-3f5e361017fa"]) @@ -89,9 +97,58 @@ def test_save_dataset_invalid_tags(mock_dependencies, tags): df=MagicMock(spec=DataFrame), dataset_name="name", dataset_description="description", + schema_id=uuid.uuid4(), tags=tags ) +@pytest.mark.parametrize( + "identifier", + [None, "5f69754e-37a0-431b-aa3e-3f5e361017fa", 123], +) +def test_save_dataset_invalid_schema_version_id(mock_dependencies, identifier): + datacatalogue, _, _ = mock_dependencies + + with pytest.raises(ValueError if identifier is None else TypeError): + datacatalogue.save( + df=MagicMock(spec=DataFrame), + dataset_name="name", + dataset_description="description", + schema_version_id=identifier, + ) + + +def test_save_dataset_rejects_both_schema_identifiers(mock_dependencies): + datacatalogue, _, _ = mock_dependencies + + with pytest.raises(ValueError, match="cannot both be provided"): + datacatalogue.save( + df=MagicMock(spec=DataFrame), + dataset_name="name", + dataset_description="description", + schema_id=uuid.uuid4(), + schema_version_id=uuid.uuid4(), + ) + + +@patch.object(DeltaStorage, "write", return_value="mock_write_success") +@patch.object(Gateway, "import_dataset", return_value="mock_datacatalog_success") +def test_save_dataset_with_schema_version_id( + mock_gateway_import, mock_write, mock_dependencies +): + datacatalogue, _, _ = mock_dependencies + schema_version_id = uuid.uuid4() + + result = datacatalogue.save( + df=MagicMock(spec=DataFrame), + dataset_name="test_dataset", + dataset_description="test description", + schema_version_id=schema_version_id, + ) + + assert result == "mock_datacatalog_success" + mock_write.assert_called_once() + assert mock_gateway_import.call_args.kwargs["schema_version_id"] == schema_version_id + def test_save_gateway_failure_triggers_rollback(mock_dependencies, storage_handler): @@ -110,7 +167,8 @@ def test_save_gateway_failure_triggers_rollback(mock_dependencies, storage_handl result = datacatalogue.save( df=MagicMock(spec=DataFrame), dataset_name="name", - dataset_description="description" + dataset_description="description", + schema_id=uuid.uuid4(), ) assert "error" in result @@ -138,7 +196,8 @@ def test_save_gateway_failure_and_rollback_fails(mock_dependencies, storage_hand result = datacatalogue.save( df=MagicMock(spec=DataFrame), dataset_name="name", - dataset_description="description" + dataset_description="description", + schema_id=uuid.uuid4(), ) assert "error" in result diff --git a/tests/test_gateway.py b/tests/test_gateway.py index c7c29fe..54c93cf 100644 --- a/tests/test_gateway.py +++ b/tests/test_gateway.py @@ -4,6 +4,7 @@ from dataworkbench.gateway import Gateway from requests.exceptions import RequestException import json +import uuid @pytest.fixture def mock_gateway(): @@ -32,6 +33,31 @@ def test_import_dataset_success(mock_gateway, mock_post): assert result == {"status": "success"} mock_post.assert_called_once() + payload = mock_post.call_args.kwargs["json"] + assert payload["schemaId"] == "schema_id" + assert "schemaVersionId" not in payload + + +def test_import_dataset_with_schema_version_id(mock_gateway, mock_post): + mock_response = MagicMock() + mock_response.json.return_value = {"status": "success"} + mock_response.raise_for_status = MagicMock() + mock_post.return_value = mock_response + schema_version_id = uuid.uuid4() + + result = mock_gateway.import_dataset( + "dataset_name", + "dataset_description", + None, + {"tag": "value"}, + "folder_id", + schema_version_id=schema_version_id, + ) + + assert result == {"status": "success"} + payload = mock_post.call_args.kwargs["json"] + assert payload["schemaVersionId"] == str(schema_version_id) + assert "schemaId" not in payload def test_import_dataset_failure(mock_gateway, mock_post):