From 442f7828d9eb14f9d7eee695a102902a5b133411 Mon Sep 17 00:00:00 2001
From: Lina
Date: Mon, 9 Nov 2026 01:04:16 +0800
Subject: [PATCH 1/2] #809469 check the schema id
---
src/dataworkbench/datacatalogue.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/src/dataworkbench/datacatalogue.py b/src/dataworkbench/datacatalogue.py
index e6f2176..c5f9683 100644
--- a/src/dataworkbench/datacatalogue.py
+++ b/src/dataworkbench/datacatalogue.py
@@ -70,7 +70,7 @@ def save(
df: DataFrame,
dataset_name: str,
dataset_description: str,
- schema_id: uuid.UUID | None = None,
+ schema_id: uuid.UUID,
tags: dict[str, str] | None = None,
) -> dict[str, Any]:
"""
@@ -84,7 +84,7 @@ def save(
df: Spark DataFrame containing the dataset to be saved
dataset_name: Name of the dataset for the catalog
dataset_description: Description of the dataset's purpose and contents
- schema_id: Optional UUID for the dataset's schema
+ schema_id: UUID for the dataset's schema
tags: Optional metadata tags as key-value pairs
Returns:
@@ -102,7 +102,7 @@ def save(
... df,
... "Customer Data",
... "Monthly customer demographic data",
- ... "550e8400-e29b-41d4-a716-446655440000",
+ ... uuid.UUID("550e8400-e29b-41d4-a716-446655440000"),
... {"environment": ["test"]}
... )
"""
@@ -116,6 +116,9 @@ def save(
if not isinstance(dataset_description, str):
raise TypeError("dataset_description must be a string")
+ if not isinstance(schema_id, uuid.UUID) or not schema_id:
+ raise TypeError("schema_id must be a non-empty UUID")
+
if tags is not None and not isinstance(tags, dict):
raise TypeError("tags must be a dictionary or None")
From 54acb38f4bf4dd31ab7166324f4e3a50ab8fda49 Mon Sep 17 00:00:00 2001
From: Lina
Date: Mon, 9 Nov 2026 01:32:45 +0800
Subject: [PATCH 2/2] #809469 support either schema id or schema version id
---
README.md | 21 ++++++++-
src/dataworkbench/datacatalogue.py | 38 ++++++++++++----
src/dataworkbench/gateway.py | 11 ++++-
tests/test_datacatalogue.py | 71 +++++++++++++++++++++++++++---
tests/test_gateway.py | 26 +++++++++++
5 files changed, 149 insertions(+), 18 deletions(-)
diff --git a/README.md b/README.md
index 3269962..079bb33 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,8 @@ datacatalogue.save(
#### Using an existing schema
When you have an existing schema that you want to reuse:
```python
+import uuid
+
from dataworkbench import DataCatalogue
df = spark.createDataFrame([("a", 1), ("b", 2), ("c", 3)], ["letter", "number"])
@@ -82,15 +84,30 @@ datacatalogue.save(
"Dataset Name",
"Description",
tags={"environment": ["test"]},
- schema_id="abada0f7-acb4-43cf-8f54-b51abd7ba8b1" # Using an existing schema ID
+ schema_id=uuid.UUID("abada0f7-acb4-43cf-8f54-b51abd7ba8b1"),
)
```
+#### Using an existing schema version
+When a specific schema version is required:
+```python
+import uuid
+
+datacatalogue.save(
+ df,
+ "Dataset Name",
+ "Description",
+ schema_version_id=uuid.UUID("abada0f7-acb4-43cf-8f54-b51abd7ba8b1"),
+ tags={"environment": ["test"]},
+)
+```
+
+Provide exactly one of `schema_id` or `schema_version_id`.
## API Reference
### DataCatalogue
-- `save(df, name, description, schema_id=None, tags=None)`: Save a Spark DataFrame to the Data Workbench Data Catalogue
+- `save(df, name, description, schema_id=None, tags=None, schema_version_id=None)`: Save a Spark DataFrame to the Data Workbench Data Catalogue. Provide exactly one schema identifier.
## License
diff --git a/src/dataworkbench/datacatalogue.py b/src/dataworkbench/datacatalogue.py
index c5f9683..908efee 100644
--- a/src/dataworkbench/datacatalogue.py
+++ b/src/dataworkbench/datacatalogue.py
@@ -70,8 +70,9 @@ def save(
df: DataFrame,
dataset_name: str,
dataset_description: str,
- schema_id: uuid.UUID,
+ schema_id: uuid.UUID | None = None,
tags: dict[str, str] | None = None,
+ schema_version_id: uuid.UUID | None = None,
) -> dict[str, Any]:
"""
Save a dataset to storage and register it with the data catalog.
@@ -84,8 +85,11 @@ def save(
df: Spark DataFrame containing the dataset to be saved
dataset_name: Name of the dataset for the catalog
dataset_description: Description of the dataset's purpose and contents
- schema_id: UUID for the dataset's schema
+ schema_id: UUID for the dataset's schema. Mutually exclusive with
+ schema_version_id.
tags: Optional metadata tags as key-value pairs
+ schema_version_id: UUID for the dataset schema version. Mutually
+ exclusive with schema_id.
Returns:
Dict[str, Any]: Response from the Gateway service containing dataset
@@ -102,8 +106,10 @@ def save(
... df,
... "Customer Data",
... "Monthly customer demographic data",
- ... uuid.UUID("550e8400-e29b-41d4-a716-446655440000"),
- ... {"environment": ["test"]}
+ ... schema_version_id=uuid.UUID(
+ ... "550e8400-e29b-41d4-a716-446655440000"
+ ... ),
+ ... tags={"environment": ["test"]},
... )
"""
# Validate input parameters
@@ -116,12 +122,23 @@ def save(
if not isinstance(dataset_description, str):
raise TypeError("dataset_description must be a string")
- if not isinstance(schema_id, uuid.UUID) or not schema_id:
- raise TypeError("schema_id must be a non-empty UUID")
-
if tags is not None and not isinstance(tags, dict):
raise TypeError("tags must be a dictionary or None")
+ if schema_id is None and schema_version_id is None:
+ raise ValueError("Either schema_id or schema_version_id must be provided")
+
+ if schema_id is not None and schema_version_id is not None:
+ raise ValueError("schema_id and schema_version_id cannot both be provided")
+
+ if schema_id is not None and not isinstance(schema_id, uuid.UUID):
+ raise TypeError("schema_id must be a UUID")
+
+ if schema_version_id is not None and not isinstance(
+ schema_version_id, uuid.UUID
+ ):
+ raise TypeError("schema_version_id must be a UUID")
+
# Generate folder_id
folder_id = uuid.uuid4()
@@ -134,7 +151,12 @@ def save(
try:
# Register the dataset with the Gateway API
return self.gateway.import_dataset(
- dataset_name, dataset_description, schema_id, tags or {}, folder_id
+ dataset_name,
+ dataset_description,
+ schema_id,
+ tags or {},
+ folder_id,
+ schema_version_id=schema_version_id,
)
except Exception as e:
self._rollback_write(folder_id)
diff --git a/src/dataworkbench/gateway.py b/src/dataworkbench/gateway.py
index bc989b6..911766f 100644
--- a/src/dataworkbench/gateway.py
+++ b/src/dataworkbench/gateway.py
@@ -111,6 +111,7 @@ def import_dataset(
schema_id: UUID | None,
tags: dict[str, str],
folder_id: UUID,
+ schema_version_id: UUID | None = None,
) -> dict[str, Any]:
"""
Imports a dataset into the DataWorkbench.
@@ -120,9 +121,12 @@ def import_dataset(
Args:
dataset_name: Name of the dataset to be imported
dataset_description: Description of the dataset's purpose and contents
- schema_id: UUID identifier for the dataset's schema
+ schema_id: UUID identifier for the dataset's schema. Mutually
+ exclusive with schema_version_id.
tags: Metadata tags as key-value pairs for categorization and discovery
folder_id: UUID identifier for the folder where the dataset will be stored
+ schema_version_id: UUID identifier for the dataset's schema version.
+ Mutually exclusive with schema_id.
Returns:
Dict[str, Any]: JSON response from the API or an error message dictionary
@@ -144,9 +148,12 @@ def import_dataset(
"datasetFolderId": str(folder_id),
"datasetName": dataset_name,
"datasetDescription": dataset_description,
- "schemaId": str(schema_id) if schema_id else schema_id,
"tags": tags,
}
+ if schema_id is not None:
+ payload["schemaId"] = str(schema_id)
+ if schema_version_id is not None:
+ payload["schemaVersionId"] = str(schema_version_id)
logger.info(f"Sending request to import dataset: {dataset_name}")
diff --git a/tests/test_datacatalogue.py b/tests/test_datacatalogue.py
index 2bf7a75..47b0d9c 100644
--- a/tests/test_datacatalogue.py
+++ b/tests/test_datacatalogue.py
@@ -31,19 +31,27 @@ def storage_handler():
@patch.object(DeltaStorage, "write", return_value="mock_write_success")
@patch.object(Gateway, "import_dataset", return_value="mock_datacatalog_success")
-def test_save_dataset(mock_write, mock_gateway_import, mock_dependencies):
+def test_save_dataset(mock_gateway_import, mock_write, mock_dependencies):
"""Test saving a dataset without making real API calls"""
datacatalogue, _, _ = mock_dependencies
+ schema_id = uuid.uuid4()
+ tags = {"environment": "test"}
result = datacatalogue.save(
- df=MagicMock(spec=DataFrame),
- dataset_name="test_dataset",
- dataset_description="test description"
+ MagicMock(spec=DataFrame),
+ "test_dataset",
+ "test description",
+ schema_id,
+ tags,
)
assert result == "mock_datacatalog_success"
mock_write.assert_called_once()
mock_gateway_import.assert_called_once()
+ gateway_args = mock_gateway_import.call_args.args
+ assert gateway_args[:4] == ("test_dataset", "test description", schema_id, tags)
+ assert isinstance(gateway_args[4], uuid.UUID)
+ assert mock_gateway_import.call_args.kwargs == {"schema_version_id": None}
@pytest.mark.parametrize("folder_id", ["", 123, "5f69754e-37a0-431b-aa3e-3f5e361017fa"])
@@ -89,9 +97,58 @@ def test_save_dataset_invalid_tags(mock_dependencies, tags):
df=MagicMock(spec=DataFrame),
dataset_name="name",
dataset_description="description",
+ schema_id=uuid.uuid4(),
tags=tags
)
+@pytest.mark.parametrize(
+ "identifier",
+ [None, "5f69754e-37a0-431b-aa3e-3f5e361017fa", 123],
+)
+def test_save_dataset_invalid_schema_version_id(mock_dependencies, identifier):
+ datacatalogue, _, _ = mock_dependencies
+
+ with pytest.raises(ValueError if identifier is None else TypeError):
+ datacatalogue.save(
+ df=MagicMock(spec=DataFrame),
+ dataset_name="name",
+ dataset_description="description",
+ schema_version_id=identifier,
+ )
+
+
+def test_save_dataset_rejects_both_schema_identifiers(mock_dependencies):
+ datacatalogue, _, _ = mock_dependencies
+
+ with pytest.raises(ValueError, match="cannot both be provided"):
+ datacatalogue.save(
+ df=MagicMock(spec=DataFrame),
+ dataset_name="name",
+ dataset_description="description",
+ schema_id=uuid.uuid4(),
+ schema_version_id=uuid.uuid4(),
+ )
+
+
+@patch.object(DeltaStorage, "write", return_value="mock_write_success")
+@patch.object(Gateway, "import_dataset", return_value="mock_datacatalog_success")
+def test_save_dataset_with_schema_version_id(
+ mock_gateway_import, mock_write, mock_dependencies
+):
+ datacatalogue, _, _ = mock_dependencies
+ schema_version_id = uuid.uuid4()
+
+ result = datacatalogue.save(
+ df=MagicMock(spec=DataFrame),
+ dataset_name="test_dataset",
+ dataset_description="test description",
+ schema_version_id=schema_version_id,
+ )
+
+ assert result == "mock_datacatalog_success"
+ mock_write.assert_called_once()
+ assert mock_gateway_import.call_args.kwargs["schema_version_id"] == schema_version_id
+
def test_save_gateway_failure_triggers_rollback(mock_dependencies, storage_handler):
@@ -110,7 +167,8 @@ def test_save_gateway_failure_triggers_rollback(mock_dependencies, storage_handl
result = datacatalogue.save(
df=MagicMock(spec=DataFrame),
dataset_name="name",
- dataset_description="description"
+ dataset_description="description",
+ schema_id=uuid.uuid4(),
)
assert "error" in result
@@ -138,7 +196,8 @@ def test_save_gateway_failure_and_rollback_fails(mock_dependencies, storage_hand
result = datacatalogue.save(
df=MagicMock(spec=DataFrame),
dataset_name="name",
- dataset_description="description"
+ dataset_description="description",
+ schema_id=uuid.uuid4(),
)
assert "error" in result
diff --git a/tests/test_gateway.py b/tests/test_gateway.py
index c7c29fe..54c93cf 100644
--- a/tests/test_gateway.py
+++ b/tests/test_gateway.py
@@ -4,6 +4,7 @@
from dataworkbench.gateway import Gateway
from requests.exceptions import RequestException
import json
+import uuid
@pytest.fixture
def mock_gateway():
@@ -32,6 +33,31 @@ def test_import_dataset_success(mock_gateway, mock_post):
assert result == {"status": "success"}
mock_post.assert_called_once()
+ payload = mock_post.call_args.kwargs["json"]
+ assert payload["schemaId"] == "schema_id"
+ assert "schemaVersionId" not in payload
+
+
+def test_import_dataset_with_schema_version_id(mock_gateway, mock_post):
+ mock_response = MagicMock()
+ mock_response.json.return_value = {"status": "success"}
+ mock_response.raise_for_status = MagicMock()
+ mock_post.return_value = mock_response
+ schema_version_id = uuid.uuid4()
+
+ result = mock_gateway.import_dataset(
+ "dataset_name",
+ "dataset_description",
+ None,
+ {"tag": "value"},
+ "folder_id",
+ schema_version_id=schema_version_id,
+ )
+
+ assert result == {"status": "success"}
+ payload = mock_post.call_args.kwargs["json"]
+ assert payload["schemaVersionId"] == str(schema_version_id)
+ assert "schemaId" not in payload
def test_import_dataset_failure(mock_gateway, mock_post):