Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ datacatalogue.save(
#### Using an existing schema
When you have an existing schema that you want to reuse:
```python
import uuid

from dataworkbench import DataCatalogue

df = spark.createDataFrame([("a", 1), ("b", 2), ("c", 3)], ["letter", "number"])
Expand All @@ -82,15 +84,30 @@ datacatalogue.save(
"Dataset Name",
"Description",
tags={"environment": ["test"]},
schema_id="abada0f7-acb4-43cf-8f54-b51abd7ba8b1" # Using an existing schema ID
schema_id=uuid.UUID("abada0f7-acb4-43cf-8f54-b51abd7ba8b1"),
)
```
#### Using an existing schema version
When a specific schema version is required:
```python
import uuid

datacatalogue.save(
df,
"Dataset Name",
"Description",
schema_version_id=uuid.UUID("abada0f7-acb4-43cf-8f54-b51abd7ba8b1"),
tags={"environment": ["test"]},
)
```

Provide exactly one of `schema_id` or `schema_version_id`.

## API Reference

### DataCatalogue

- `save(df, name, description, schema_id=None, tags=None)`: Save a Spark DataFrame to the Data Workbench Data Catalogue
- `save(df, name, description, schema_id=None, tags=None, schema_version_id=None)`: Save a Spark DataFrame to the Data Workbench Data Catalogue. Provide exactly one schema identifier.


## License
Expand Down
33 changes: 29 additions & 4 deletions src/dataworkbench/datacatalogue.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ def save(
dataset_description: str,
schema_id: uuid.UUID | None = None,
tags: dict[str, str] | None = None,
schema_version_id: uuid.UUID | None = None,
) -> dict[str, Any]:
"""
Save a dataset to storage and register it with the data catalog.
Expand All @@ -84,8 +85,11 @@ def save(
df: Spark DataFrame containing the dataset to be saved
dataset_name: Name of the dataset for the catalog
dataset_description: Description of the dataset's purpose and contents
schema_id: Optional UUID for the dataset's schema
schema_id: UUID for the dataset's schema. Mutually exclusive with
schema_version_id.
tags: Optional metadata tags as key-value pairs
schema_version_id: UUID for the dataset schema version. Mutually
exclusive with schema_id.

Returns:
Dict[str, Any]: Response from the Gateway service containing dataset
Expand All @@ -102,8 +106,10 @@ def save(
... df,
... "Customer Data",
... "Monthly customer demographic data",
... "550e8400-e29b-41d4-a716-446655440000",
... {"environment": ["test"]}
... schema_version_id=uuid.UUID(
... "550e8400-e29b-41d4-a716-446655440000"
... ),
... tags={"environment": ["test"]},
... )
"""
# Validate input parameters
Expand All @@ -119,6 +125,20 @@ def save(
if tags is not None and not isinstance(tags, dict):
raise TypeError("tags must be a dictionary or None")

if schema_id is None and schema_version_id is None:
raise ValueError("Either schema_id or schema_version_id must be provided")

if schema_id is not None and schema_version_id is not None:
raise ValueError("schema_id and schema_version_id cannot both be provided")

if schema_id is not None and not isinstance(schema_id, uuid.UUID):
raise TypeError("schema_id must be a UUID")

if schema_version_id is not None and not isinstance(
schema_version_id, uuid.UUID
):
raise TypeError("schema_version_id must be a UUID")

# Generate folder_id
folder_id = uuid.uuid4()

Expand All @@ -131,7 +151,12 @@ def save(
try:
# Register the dataset with the Gateway API
return self.gateway.import_dataset(
dataset_name, dataset_description, schema_id, tags or {}, folder_id
dataset_name,
dataset_description,
schema_id,
tags or {},
folder_id,
schema_version_id=schema_version_id,
)
except Exception as e:
self._rollback_write(folder_id)
Expand Down
11 changes: 9 additions & 2 deletions src/dataworkbench/gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ def import_dataset(
schema_id: UUID | None,
tags: dict[str, str],
folder_id: UUID,
schema_version_id: UUID | None = None,
) -> dict[str, Any]:
"""
Imports a dataset into the DataWorkbench.
Expand All @@ -120,9 +121,12 @@ def import_dataset(
Args:
dataset_name: Name of the dataset to be imported
dataset_description: Description of the dataset's purpose and contents
schema_id: UUID identifier for the dataset's schema
schema_id: UUID identifier for the dataset's schema. Mutually
exclusive with schema_version_id.
tags: Metadata tags as key-value pairs for categorization and discovery
folder_id: UUID identifier for the folder where the dataset will be stored
schema_version_id: UUID identifier for the dataset's schema version.
Mutually exclusive with schema_id.

Returns:
Dict[str, Any]: JSON response from the API or an error message dictionary
Expand All @@ -144,9 +148,12 @@ def import_dataset(
"datasetFolderId": str(folder_id),
"datasetName": dataset_name,
"datasetDescription": dataset_description,
"schemaId": str(schema_id) if schema_id else schema_id,
"tags": tags,
}
if schema_id is not None:
payload["schemaId"] = str(schema_id)
if schema_version_id is not None:
payload["schemaVersionId"] = str(schema_version_id)

logger.info(f"Sending request to import dataset: {dataset_name}")

Expand Down
71 changes: 65 additions & 6 deletions tests/test_datacatalogue.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,27 @@ def storage_handler():

@patch.object(DeltaStorage, "write", return_value="mock_write_success")
@patch.object(Gateway, "import_dataset", return_value="mock_datacatalog_success")
def test_save_dataset(mock_write, mock_gateway_import, mock_dependencies):
def test_save_dataset(mock_gateway_import, mock_write, mock_dependencies):
"""Test saving a dataset without making real API calls"""
datacatalogue, _, _ = mock_dependencies
schema_id = uuid.uuid4()
tags = {"environment": "test"}

result = datacatalogue.save(
df=MagicMock(spec=DataFrame),
dataset_name="test_dataset",
dataset_description="test description"
MagicMock(spec=DataFrame),
"test_dataset",
"test description",
schema_id,
tags,
)

assert result == "mock_datacatalog_success"
mock_write.assert_called_once()
mock_gateway_import.assert_called_once()
gateway_args = mock_gateway_import.call_args.args
assert gateway_args[:4] == ("test_dataset", "test description", schema_id, tags)
assert isinstance(gateway_args[4], uuid.UUID)
assert mock_gateway_import.call_args.kwargs == {"schema_version_id": None}


@pytest.mark.parametrize("folder_id", ["", 123, "5f69754e-37a0-431b-aa3e-3f5e361017fa"])
Expand Down Expand Up @@ -89,9 +97,58 @@ def test_save_dataset_invalid_tags(mock_dependencies, tags):
df=MagicMock(spec=DataFrame),
dataset_name="name",
dataset_description="description",
schema_id=uuid.uuid4(),
tags=tags
)

@pytest.mark.parametrize(
"identifier",
[None, "5f69754e-37a0-431b-aa3e-3f5e361017fa", 123],
)
def test_save_dataset_invalid_schema_version_id(mock_dependencies, identifier):
datacatalogue, _, _ = mock_dependencies

with pytest.raises(ValueError if identifier is None else TypeError):
datacatalogue.save(
df=MagicMock(spec=DataFrame),
dataset_name="name",
dataset_description="description",
schema_version_id=identifier,
)


def test_save_dataset_rejects_both_schema_identifiers(mock_dependencies):
datacatalogue, _, _ = mock_dependencies

with pytest.raises(ValueError, match="cannot both be provided"):
datacatalogue.save(
df=MagicMock(spec=DataFrame),
dataset_name="name",
dataset_description="description",
schema_id=uuid.uuid4(),
schema_version_id=uuid.uuid4(),
)


@patch.object(DeltaStorage, "write", return_value="mock_write_success")
@patch.object(Gateway, "import_dataset", return_value="mock_datacatalog_success")
def test_save_dataset_with_schema_version_id(
mock_gateway_import, mock_write, mock_dependencies
):
datacatalogue, _, _ = mock_dependencies
schema_version_id = uuid.uuid4()

result = datacatalogue.save(
df=MagicMock(spec=DataFrame),
dataset_name="test_dataset",
dataset_description="test description",
schema_version_id=schema_version_id,
)

assert result == "mock_datacatalog_success"
mock_write.assert_called_once()
assert mock_gateway_import.call_args.kwargs["schema_version_id"] == schema_version_id


def test_save_gateway_failure_triggers_rollback(mock_dependencies, storage_handler):

Expand All @@ -110,7 +167,8 @@ def test_save_gateway_failure_triggers_rollback(mock_dependencies, storage_handl
result = datacatalogue.save(
df=MagicMock(spec=DataFrame),
dataset_name="name",
dataset_description="description"
dataset_description="description",
schema_id=uuid.uuid4(),
)

assert "error" in result
Expand Down Expand Up @@ -138,7 +196,8 @@ def test_save_gateway_failure_and_rollback_fails(mock_dependencies, storage_hand
result = datacatalogue.save(
df=MagicMock(spec=DataFrame),
dataset_name="name",
dataset_description="description"
dataset_description="description",
schema_id=uuid.uuid4(),
)

assert "error" in result
Expand Down
26 changes: 26 additions & 0 deletions tests/test_gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from dataworkbench.gateway import Gateway
from requests.exceptions import RequestException
import json
import uuid

@pytest.fixture
def mock_gateway():
Expand Down Expand Up @@ -32,6 +33,31 @@ def test_import_dataset_success(mock_gateway, mock_post):

assert result == {"status": "success"}
mock_post.assert_called_once()
payload = mock_post.call_args.kwargs["json"]
assert payload["schemaId"] == "schema_id"
assert "schemaVersionId" not in payload


def test_import_dataset_with_schema_version_id(mock_gateway, mock_post):
mock_response = MagicMock()
mock_response.json.return_value = {"status": "success"}
mock_response.raise_for_status = MagicMock()
mock_post.return_value = mock_response
schema_version_id = uuid.uuid4()

result = mock_gateway.import_dataset(
"dataset_name",
"dataset_description",
None,
{"tag": "value"},
"folder_id",
schema_version_id=schema_version_id,
)

assert result == {"status": "success"}
payload = mock_post.call_args.kwargs["json"]
assert payload["schemaVersionId"] == str(schema_version_id)
assert "schemaId" not in payload


def test_import_dataset_failure(mock_gateway, mock_post):
Expand Down
Loading