diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index bf73ed4f56bfa..06279476724d6 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -210,13 +210,21 @@ String ContainerClientWrapper::GetBlobPath(const String & blob_name) const "Failed to parse Azure connection string or blob URL: {}", e.what()); } +/// The connection string the clients are created from. A disk can set it together with an `endpoint`, whose host +/// the clients then do not use. The default-constructed `auth_method` of a SAS or catalog path is empty. +static const String & getClientConnectionString(const ConnectionParams & params) +{ + const auto & connection_string = std::get(params.auth_method).toUnderType(); + return connection_string.empty() ? params.endpoint.storage_account_url : connection_string; +} + String ConnectionParams::getConnectionURL() const { if (std::holds_alternative(auth_method)) { try { - auto parsed_connection_string = Azure::Storage::_internal::ParseConnectionString(endpoint.storage_account_url); + auto parsed_connection_string = Azure::Storage::_internal::ParseConnectionString(getClientConnectionString(*this)); return parsed_connection_string.BlobServiceUrl.GetAbsoluteUrl(); } catch (const std::logic_error & e) @@ -387,6 +395,8 @@ std::unique_ptr getContainerClient(const ConnectionParams & par ProfileEvents::increment(ProfileEvents::DiskAzureCreateContainer); auto raw_client = service_client->CreateBlobContainer(params.endpoint.container_name).Value; + /// `CreateBlobContainer` returns a client without `ClickhouseOptions`. + raw_client.SetClickhouseOptions(params.client_options.ClickhouseOptions); return std::make_unique(std::move(raw_client), params.endpoint.prefix); } catch (const Azure::Storage::StorageException & e) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp index 64eb30b089153..a1c63b02a460c 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp @@ -14,6 +14,7 @@ #if USE_AZURE_BLOB_STORAGE #include #include +#include #endif #include @@ -102,6 +103,32 @@ ObjectStoragePtr ObjectStorageFactory::create( return it->second(name, config, config_prefix, context, skip_access_check, remote_host_filter_check); } +/// A custom disk with an endpoint that remote_url_allow_hosts does not allow: a log line per disk +/// and one system.warnings entry that lists all such disks. `endpoint` is what `allow_list_entry` +/// (an or element) has to allow. +[[maybe_unused]] static void warnAboutDiskNotAllowedByRemoteHostFilter( + const std::string & disk_name, const std::string & endpoint, const std::string & allow_list_entry, const ContextPtr & global_context) +{ + LOG_WARNING( + getLogger("ObjectStorageFactory"), + "Custom disk `{}` uses endpoint `{}` that is not allowed by `remote_url_allow_hosts`. The disk was created before " + "this check and is kept while loading existing metadata (server start or UNDROP). Add `{}` " + "to the list or recreate the tables that use it.", + disk_name, endpoint, allow_list_entry); + + /// system.warnings keeps one message per type, so the message lists all disks seen so far. + /// No lock: disk creation runs under storage_policies_mutex in Context::getOrCreateDisk. + static std::set disks; + disks.insert(fmt::format("`{}` -> `{}`", disk_name, endpoint)); + global_context->addOrUpdateWarningMessage( + Context::WarningType::CUSTOM_DISK_NOT_ALLOWED_BY_REMOTE_URL_ALLOW_HOSTS, + PreformattedMessage::create( + "Custom disks with endpoints that are not allowed by `remote_url_allow_hosts` were kept while loading existing " + "metadata (server start or UNDROP): {}. Add `host/bucket` (S3) or `host` " + "(Azure) entries for them or recreate the tables that use them.", + fmt::join(disks, ", "))); +} + #if USE_AWS_S3 namespace { @@ -132,35 +159,14 @@ static std::string getEndpoint( return context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); } -/// A custom disk with an endpoint that remote_url_allow_hosts does not allow: a log line per disk -/// and one system.warnings entry that lists all such disks. -static void warnAboutDiskNotAllowedByRemoteHostFilter(const std::string & disk_name, const S3::URI & uri, const ContextPtr & global_context) +/// host[:port]/bucket of an S3 endpoint, as in an entry. +static std::string getS3BucketEntry(const S3::URI & uri) { Poco::URI endpoint_uri(uri.endpoint); - /// host[:port], as in an entry. std::string host_and_port = endpoint_uri.getAuthority(); if (auto at = host_and_port.rfind('@'); at != std::string::npos) host_and_port.erase(0, at + 1); - const std::string s3_bucket_entry = host_and_port + "/" + uri.bucket; - - LOG_WARNING( - getLogger("ObjectStorageFactory"), - "Custom disk `{}` uses endpoint `{}` that is not allowed by `remote_url_allow_hosts`. The disk was created before " - "this check and is kept while loading existing metadata (server start or UNDROP). Add `{}` " - "to the list or recreate the tables that use it.", - disk_name, s3_bucket_entry, s3_bucket_entry); - - /// system.warnings keeps one message per type, so the message lists all disks seen so far. - /// No lock: disk creation runs under storage_policies_mutex in Context::getOrCreateDisk. - static std::set disks; - disks.insert(fmt::format("`{}` -> `{}`", disk_name, s3_bucket_entry)); - global_context->addOrUpdateWarningMessage( - Context::WarningType::CUSTOM_DISK_NOT_ALLOWED_BY_REMOTE_URL_ALLOW_HOSTS, - PreformattedMessage::create( - "Custom disks with endpoints that are not allowed by `remote_url_allow_hosts` were kept while loading existing " - "metadata (server start or UNDROP): {}. Add `host/bucket` entries for them or recreate " - "the tables that use them.", - fmt::join(disks, ", "))); + return host_and_port + "/" + uri.bucket; } static void registerS3ObjectStorage(ObjectStorageFactory & factory) @@ -186,7 +192,11 @@ static void registerS3ObjectStorage(ObjectStorageFactory & factory) && !uri.checkRemoteHostFilter( context->getGlobalContext()->getRemoteHostFilter(), /* throw_on_error */ remote_host_filter_check == RemoteHostFilterCheck::Enforce)) - warnAboutDiskNotAllowedByRemoteHostFilter(name, uri, context->getGlobalContext()); + { + const auto s3_bucket_entry = getS3BucketEntry(uri); + warnAboutDiskNotAllowedByRemoteHostFilter( + name, s3_bucket_entry, fmt::format("{}", s3_bucket_entry), context->getGlobalContext()); + } auto client = getClient(endpoint, *settings, context, /* for_disk_s3 */ true, name); auto key_generator = getKeyGenerator(uri, config, config_prefix); @@ -235,7 +245,7 @@ static void registerAzureObjectStorage(ObjectStorageFactory & factory) const std::string & config_prefix, const ContextPtr & context, bool /* skip_access_check */, - RemoteHostFilterCheck /* remote_host_filter_check */) -> ObjectStoragePtr + RemoteHostFilterCheck remote_host_filter_check) -> ObjectStoragePtr { auto azure_settings = AzureBlobStorage::getRequestSettings(config, config_prefix, context->getSettingsRef()); @@ -252,6 +262,26 @@ static void registerAzureObjectStorage(ObjectStorageFactory & factory) .client_options = AzureBlobStorage::getClientOptions(context, context->getSettingsRef(), *azure_settings, /*for_disk=*/ true) }; + /// As for S3: a disk loaded from existing metadata is only reported, so that it fails neither + /// the table load nor the server start. + if (remote_host_filter_check != RemoteHostFilterCheck::None) + { + auto check_host = [&](const String & url) + { + Poco::URI uri(url); + /// The query can carry a SAS token, which the error must not quote. + uri.setPathEtc(""); + if (!context->getRemoteHostFilter().checkURL( + uri, /* throw_on_error */ remote_host_filter_check == RemoteHostFilterCheck::Enforce)) + warnAboutDiskNotAllowedByRemoteHostFilter( + name, uri.getAuthority(), fmt::format("{}", uri.getHost()), context->getGlobalContext()); + }; + check_host(params.getConnectionURL()); + /// Writes to an ADLS Gen2 (OneLake) endpoint go to another host. + if (isAdlsGen2Endpoint(params.endpoint)) + check_host(getAdlsGen2Url(params.endpoint, params.auth_method)); + } + return std::make_shared( name, params.auth_method, AzureBlobStorage::getContainerClient(params, /*readonly=*/ false), std::move(azure_settings), diff --git a/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.cpp b/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.cpp index 6102225ee53cd..ee4b3a91b9fe7 100644 --- a/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.cpp @@ -13,6 +13,7 @@ #include #include +#include #include @@ -112,6 +113,14 @@ Azure::Storage::Files::DataLake::DataLakeFileClient makeAdlsGen2FileClient( auth_method); } +String getAdlsGen2Url(const AzureBlobStorage::Endpoint & endpoint, const AzureBlobStorage::AuthMethod & auth_method) +{ + const auto * connection_string = std::get_if(&auth_method); + if (connection_string && endpoint.sas_auth.empty() && endpoint.additional_params.empty()) + return Azure::Storage::_internal::ParseConnectionString(connection_string->toUnderType()).DataLakeServiceUrl.GetAbsoluteUrl(); + return buildAdlsGen2FileUrl(endpoint, /*blob_path=*/ ""); +} + WriteBufferFromAzureDataLakeStorage::WriteBufferFromAzureDataLakeStorage( const AzureBlobStorage::Endpoint & endpoint_, const AzureBlobStorage::AuthMethod & auth_method_, diff --git a/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.h b/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.h index be4acef948b28..1b7c2baf29719 100644 --- a/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.h @@ -75,6 +75,9 @@ Azure::Storage::Files::DataLake::DataLakeFileClient makeAdlsGen2FileClient( bool isAdlsGen2Endpoint(const AzureBlobStorage::Endpoint & endpoint); +/// The URL of the service that the clients of `makeAdlsGen2FileClient` send requests to. +String getAdlsGen2Url(const AzureBlobStorage::Endpoint & endpoint, const AzureBlobStorage::AuthMethod & auth_method); + /// Build the ADLS Gen2 (DFS) write URL for a blob path. Exposed for unit testing. String buildAdlsGen2FileUrl(const AzureBlobStorage::Endpoint & endpoint, const String & blob_path); diff --git a/src/IO/AzureBlobStorage/PocoHTTPClient.cpp b/src/IO/AzureBlobStorage/PocoHTTPClient.cpp index ca63a35e32de6..575294ae1c562 100644 --- a/src/IO/AzureBlobStorage/PocoHTTPClient.cpp +++ b/src/IO/AzureBlobStorage/PocoHTTPClient.cpp @@ -370,6 +370,8 @@ std::unique_ptr PocoAzureHTTPClient::makeRequest /// validated at CREATE time (OneLake writes go to the `.dfs` host, reads to /// `.blob`). Normalize scheme and host as Poco::URI does at CREATE time (the /// scheme resolves the default port; the host is lower-cased) so both agree. + /// Disks are exempt; their redirects are still checked below. + if (!for_disk_azure) { Poco::URI request_uri; request_uri.setScheme(url.GetScheme()); diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h index b122defb4d7fe..2a11a431ff306 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -215,7 +215,9 @@ class DataLakeConfiguration : public BaseStorageConfiguration, public std::enabl void check(ContextPtr context) override { - if (ready_object_storage && ready_object_storage->getType() == ObjectStorageType::S3) + if (ready_object_storage + && (ready_object_storage->getType() == ObjectStorageType::S3 + || ready_object_storage->getType() == ObjectStorageType::Azure)) this->checkFormat(); else BaseStorageConfiguration::check(context); diff --git a/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml b/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml index 57a03003b8a5a..6d9fa832c7f9d 100644 --- a/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml +++ b/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml @@ -3,6 +3,7 @@ host:80 host:123 yandex.ru + onelake.blob.fabric.microsoft.com diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index b1786b7aa8011..23186d5fa0d02 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -67,6 +67,9 @@ def test_config_with_hosts(start_cluster): "'http://127.0.0.1:1', 'container', 'data.csv', 'account', 'YQ==', 'CSV')", "INSERT INTO TABLE FUNCTION azureBlobStorage(" "'http://127.0.0.1:1', 'container', 'data.csv', 'account', 'YQ==', 'CSV', 'x UInt8') VALUES (1)", + "CREATE TABLE azure_blocked_disk (x UInt8) ENGINE = MergeTree ORDER BY x " + "SETTINGS disk = disk(type = azure_blob_storage, storage_account_url = 'http://127.0.0.1:1', " + "container_name = 'container', account_name = 'account', account_key = 'YQ==')", ], ) def test_azure_host_filter_before_client_creation(start_cluster, query): @@ -81,6 +84,42 @@ def test_azure_host_filter_before_client_creation(start_cluster, query): assert node5.query(container_requests) == before +@pytest.mark.parametrize( + "endpoint, rejected_url", + [ + ( + "storage_account_url = 'https://onelake.blob.fabric.microsoft.com', " + "account_name = 'account', account_key = 'YQ=='", + "https://onelake.dfs.fabric.microsoft.com", + ), + ( + "connection_string = 'BlobEndpoint=https://onelake.blob.fabric.microsoft.com/;" + "DfsEndpoint=http://127.0.0.1:1;SharedAccessSignature=sv=2022&sig=c2FzY2FuYXJ5'", + "http://127.0.0.1:1", + ), + ( + "endpoint = 'BlobEndpoint=http://host:80/account/container', " + "connection_string = 'BlobEndpoint=http://127.0.0.1:1'", + "http://127.0.0.1:1", + ), + ], +) +def test_azure_disk_client_host(start_cluster, endpoint, rejected_url): + # Each disk names a host that node2 allows, but sends requests to one that node2 does not allow. + container_requests = ( + "SELECT sum(value) FROM system.events " + "WHERE event IN ('AzureGetProperties', 'AzureCreateContainer')" + ) + before = node2.query(container_requests) + error = node2.query_and_get_error( + "CREATE TABLE azure_client_host_disk (x UInt8) ENGINE = MergeTree ORDER BY x " + f"SETTINGS disk = disk(type = azure_blob_storage, {endpoint}, container_name = 'container')", + settings={"azure_sdk_max_retries": 0}, + ) + assert f'URL "{rejected_url}" is not allowed' in error + assert node2.query(container_requests) == before + + def test_object_storage_queue_move_destination_host_filter(start_cluster): # The source is on an allowed host (`host:80`), so only the move destination can be rejected. `node1` has no # Keeper, so a rejection with "not allowed" must come from the settings validation, which runs before any diff --git a/tests/integration/test_merge_tree_azure_blob_storage/test.py b/tests/integration/test_merge_tree_azure_blob_storage/test.py index 87801c38f9b44..0eeb2a0d34027 100644 --- a/tests/integration/test_merge_tree_azure_blob_storage/test.py +++ b/tests/integration/test_merge_tree_azure_blob_storage/test.py @@ -78,6 +78,7 @@ def cluster(): path, ], with_azurite=True, + stay_alive=True, ) logging.info("Starting cluster...") cluster.start() @@ -667,6 +668,59 @@ def test_endpoint_new_container(cluster): assert 10 == int(node.query("SELECT count() FROM test")) +def test_endpoint_host_filter_on_restart(cluster): + # A query-created disk is checked against `remote_url_allow_hosts`. As for S3, a table loaded from + # existing metadata is kept with a warning when its host is no longer allowed; a new one is rejected. + node = cluster.instances[NODE_NAME] + port = cluster.azurite_port + allow_hosts_config = "/etc/clickhouse-server/config.d/remote_url_allow_hosts.xml" + + def azure_disk(container): + return f"""disk( + type = azure_blob_storage, + endpoint = 'http://azurite1:{port}/devstoreaccount1/{container}/data_prefix', + endpoint_contains_account_name = 'true', + account_name = 'devstoreaccount1', + account_key = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==')""" + + azure_query( + node, + f""" + DROP TABLE IF EXISTS test_host_filter SYNC; + + CREATE TABLE test_host_filter (a Int32) + ENGINE = MergeTree() ORDER BY tuple() + SETTINGS disk = {azure_disk('cont4')}; + + INSERT INTO test_host_filter SELECT number FROM numbers(10); + """, + ) + + try: + node.replace_config( + allow_hosts_config, + "blocked.invalid", + ) + node.restart_clickhouse() + assert 10 == int(node.query("SELECT count() FROM test_host_filter")) + assert node.contains_in_log("is not allowed by `remote_url_allow_hosts`") + assert 1 == int( + node.query( + f"SELECT count() FROM system.warnings WHERE message LIKE '%remote_url_allow_hosts%' AND message LIKE '%azurite1:{port}%'" + ) + ) + error = node.query_and_get_error( + f"CREATE TABLE test_host_filter_new (a Int32) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = {azure_disk('cont5')}" + ) + assert "UNACCEPTABLE_URL" in error, error + finally: + node.exec_in_container(["rm", "-f", allow_hosts_config]) + node.restart_clickhouse() + + assert 10 == int(node.query("SELECT count() FROM test_host_filter")) + node.query("DROP TABLE test_host_filter SYNC") + + def test_endpoint_without_prefix(cluster): node = cluster.instances[NODE_NAME] account_name = "devstoreaccount1" diff --git a/tests/integration/test_storage_iceberg_disks/test.py b/tests/integration/test_storage_iceberg_disks/test.py index 044881229f84a..a62d9bdd6336b 100644 --- a/tests/integration/test_storage_iceberg_disks/test.py +++ b/tests/integration/test_storage_iceberg_disks/test.py @@ -106,7 +106,7 @@ def generate_cluster_def(common_path, port, azure_container): disk_local_common,disk_s3_common,disk_azure_common,disk_s3_with_cache,disk_azure_with_cache - azurite1 + blocked.invalid """