From 090296c08442125f0f1f4104e5301167b611bca3 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 5 Nov 2026 14:07:18 +0000 Subject: [PATCH 1/2] Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts Do not check Azure disks from the server config against remote_url_allow_hosts Conflicts committed as-is; the next commit resolves them: - src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp (both modified) - src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp (both modified) - tests/integration/test_allowed_url_from_config/test.py (both modified) - tests/integration/test_merge_tree_azure_blob_storage/test.py (both modified) --- .../AzureBlobStorageCommon.cpp | 86 +++++++++++++++- .../ObjectStorages/ObjectStorageFactory.cpp | 97 ++++++++++++++----- .../WriteBufferFromAzureDataLakeStorage.cpp | 9 ++ .../IO/WriteBufferFromAzureDataLakeStorage.h | 3 + src/IO/AzureBlobStorage/PocoHTTPClient.cpp | 2 + .../DataLakes/DataLakeConfiguration.h | 4 +- .../config_with_only_primary_hosts.xml | 1 + .../test_allowed_url_from_config/test.py | 79 +++++++++++++++ .../test.py | 88 +++++++++++++++++ .../test_storage_iceberg_disks/test.py | 2 +- 10 files changed, 343 insertions(+), 28 deletions(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index 02a5af057f7a7..ecd9590674610 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -209,13 +209,21 @@ String ContainerClientWrapper::GetBlobPath(const String & blob_name) const "Failed to parse Azure connection string or blob URL: {}", e.what()); } +/// The connection string the clients are created from. A disk can set it together with an `endpoint`, whose host +/// the clients then do not use. The default-constructed `auth_method` of a SAS or catalog path is empty. +static const String & getClientConnectionString(const ConnectionParams & params) +{ + const auto & connection_string = std::get(params.auth_method).toUnderType(); + return connection_string.empty() ? params.endpoint.storage_account_url : connection_string; +} + String ConnectionParams::getConnectionURL() const { if (std::holds_alternative(auth_method)) { try { - auto parsed_connection_string = Azure::Storage::_internal::ParseConnectionString(endpoint.storage_account_url); + auto parsed_connection_string = Azure::Storage::_internal::ParseConnectionString(getClientConnectionString(*this)); return parsed_connection_string.BlobServiceUrl.GetAbsoluteUrl(); } catch (const std::logic_error & e) @@ -227,6 +235,80 @@ String ConnectionParams::getConnectionURL() const return endpoint.storage_account_url; } +<<<<<<< HEAD +======= +static bool isServerManagedCredential(const AuthMethod & auth_method) +{ + return std::holds_alternative>(auth_method) + || std::holds_alternative>(auth_method); +} + +[[noreturn]] static void throwServerCredentialsRestricted() +{ + throw Exception( + ErrorCodes::ACCESS_DENIED, + "Azure access from user queries is not allowed to use the server's own identity (AKS workload identity " + "or the machine's managed identity). Provide explicit credentials: an `account_name` and `account_key`, " + "a connection string, or a SAS token" +#if !CLICKHOUSE_CLOUD + ", or enable the setting `s3_allow_server_credentials_in_user_queries`" +#endif + "."); +} + +bool ConnectionParams::usesServerManagedCredentials() const +{ + /// A request authorized by SAS does not use `auth_method`. + return endpoint.sas_auth.empty() && endpoint.additional_params.empty() && isServerManagedCredential(auth_method); +} + +bool ConnectionParams::checkOrDropServerManagedCredentials() const +{ + if (!forbid_implicit_credentials || !usesServerManagedCredentials()) + return false; + + if (anonymous_fallback_for_server_credentials) + return true; + + throwServerCredentialsRestricted(); +} + +void ConnectionParams::checkSessionMayUseServerIdentity(bool restrict_server_credentials) const +{ + if (restrict_server_credentials && follows_session_restriction && !forbid_implicit_credentials && usesServerManagedCredentials()) + throwServerCredentialsRestricted(); +} + +AuthMethod ConnectionParams::getPermittedAuthMethod() const +{ + if (checkOrDropServerManagedCredentials()) + throwServerCredentialsRestricted(); + return auth_method; +} + +String ConnectionParams::getConnectionURLWithoutQuery() const +{ + if (std::holds_alternative(auth_method)) + { + try + { + /// `ParseConnectionString` appends the SAS of the connection string to `BlobServiceUrl` as its query. + auto url = Azure::Storage::_internal::ParseConnectionString(getClientConnectionString(*this)).BlobServiceUrl; + const auto query = url.GetQueryParameters(); + for (const auto & [key, value] : query) + url.RemoveQueryParameter(key); + return url.GetAbsoluteUrl(); + } + catch (const std::logic_error & e) + { + translateAzureSdkParseError(e); + } + } + + return endpoint.storage_account_url.substr(0, endpoint.storage_account_url.find_first_of("?#")); +} + +>>>>>>> fee38d30170 (Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts) std::unique_ptr ConnectionParams::createForService() const { try @@ -364,6 +446,8 @@ std::unique_ptr getContainerClient(const ConnectionParams & par ProfileEvents::increment(ProfileEvents::DiskAzureCreateContainer); auto raw_client = service_client->CreateBlobContainer(params.endpoint.container_name).Value; + /// `CreateBlobContainer` returns a client without `ClickhouseOptions`. + raw_client.SetClickhouseOptions(params.client_options.ClickhouseOptions); return std::make_unique(std::move(raw_client), params.endpoint.prefix); } catch (const Azure::Storage::StorageException & e) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp index 64eb30b089153..1b22bd6fa24a1 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp @@ -14,6 +14,11 @@ #if USE_AZURE_BLOB_STORAGE #include #include +<<<<<<< HEAD +======= +#include +#include +>>>>>>> fee38d30170 (Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts) #endif #include @@ -102,6 +107,32 @@ ObjectStoragePtr ObjectStorageFactory::create( return it->second(name, config, config_prefix, context, skip_access_check, remote_host_filter_check); } +/// A custom disk with an endpoint that remote_url_allow_hosts does not allow: a log line per disk +/// and one system.warnings entry that lists all such disks. `endpoint` is what `allow_list_entry` +/// (an or element) has to allow. +[[maybe_unused]] static void warnAboutDiskNotAllowedByRemoteHostFilter( + const std::string & disk_name, const std::string & endpoint, const std::string & allow_list_entry, const ContextPtr & global_context) +{ + LOG_WARNING( + getLogger("ObjectStorageFactory"), + "Custom disk `{}` uses endpoint `{}` that is not allowed by `remote_url_allow_hosts`. The disk was created before " + "this check and is kept while loading existing metadata (server start or UNDROP). Add `{}` " + "to the list or recreate the tables that use it.", + disk_name, endpoint, allow_list_entry); + + /// system.warnings keeps one message per type, so the message lists all disks seen so far. + /// No lock: disk creation runs under storage_policies_mutex in Context::getOrCreateDisk. + static std::set disks; + disks.insert(fmt::format("`{}` -> `{}`", disk_name, endpoint)); + global_context->addOrUpdateWarningMessage( + Context::WarningType::CUSTOM_DISK_NOT_ALLOWED_BY_REMOTE_URL_ALLOW_HOSTS, + PreformattedMessage::create( + "Custom disks with endpoints that are not allowed by `remote_url_allow_hosts` were kept while loading existing " + "metadata (server start or UNDROP): {}. Add `host/bucket` (S3) or `host` " + "(Azure) entries for them or recreate the tables that use them.", + fmt::join(disks, ", "))); +} + #if USE_AWS_S3 namespace { @@ -132,35 +163,14 @@ static std::string getEndpoint( return context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); } -/// A custom disk with an endpoint that remote_url_allow_hosts does not allow: a log line per disk -/// and one system.warnings entry that lists all such disks. -static void warnAboutDiskNotAllowedByRemoteHostFilter(const std::string & disk_name, const S3::URI & uri, const ContextPtr & global_context) +/// host[:port]/bucket of an S3 endpoint, as in an entry. +static std::string getS3BucketEntry(const S3::URI & uri) { Poco::URI endpoint_uri(uri.endpoint); - /// host[:port], as in an entry. std::string host_and_port = endpoint_uri.getAuthority(); if (auto at = host_and_port.rfind('@'); at != std::string::npos) host_and_port.erase(0, at + 1); - const std::string s3_bucket_entry = host_and_port + "/" + uri.bucket; - - LOG_WARNING( - getLogger("ObjectStorageFactory"), - "Custom disk `{}` uses endpoint `{}` that is not allowed by `remote_url_allow_hosts`. The disk was created before " - "this check and is kept while loading existing metadata (server start or UNDROP). Add `{}` " - "to the list or recreate the tables that use it.", - disk_name, s3_bucket_entry, s3_bucket_entry); - - /// system.warnings keeps one message per type, so the message lists all disks seen so far. - /// No lock: disk creation runs under storage_policies_mutex in Context::getOrCreateDisk. - static std::set disks; - disks.insert(fmt::format("`{}` -> `{}`", disk_name, s3_bucket_entry)); - global_context->addOrUpdateWarningMessage( - Context::WarningType::CUSTOM_DISK_NOT_ALLOWED_BY_REMOTE_URL_ALLOW_HOSTS, - PreformattedMessage::create( - "Custom disks with endpoints that are not allowed by `remote_url_allow_hosts` were kept while loading existing " - "metadata (server start or UNDROP): {}. Add `host/bucket` entries for them or recreate " - "the tables that use them.", - fmt::join(disks, ", "))); + return host_and_port + "/" + uri.bucket; } static void registerS3ObjectStorage(ObjectStorageFactory & factory) @@ -186,7 +196,11 @@ static void registerS3ObjectStorage(ObjectStorageFactory & factory) && !uri.checkRemoteHostFilter( context->getGlobalContext()->getRemoteHostFilter(), /* throw_on_error */ remote_host_filter_check == RemoteHostFilterCheck::Enforce)) - warnAboutDiskNotAllowedByRemoteHostFilter(name, uri, context->getGlobalContext()); + { + const auto s3_bucket_entry = getS3BucketEntry(uri); + warnAboutDiskNotAllowedByRemoteHostFilter( + name, s3_bucket_entry, fmt::format("{}", s3_bucket_entry), context->getGlobalContext()); + } auto client = getClient(endpoint, *settings, context, /* for_disk_s3 */ true, name); auto key_generator = getKeyGenerator(uri, config, config_prefix); @@ -252,6 +266,39 @@ static void registerAzureObjectStorage(ObjectStorageFactory & factory) .client_options = AzureBlobStorage::getClientOptions(context, context->getSettingsRef(), *azure_settings, /*for_disk=*/ true) }; +<<<<<<< HEAD +======= + /// As for S3: a disk loaded from existing metadata is only reported, so that it fails neither + /// the table load nor the server start. + if (remote_host_filter_check != RemoteHostFilterCheck::None) + { + auto check_host = [&](const String & url) + { + Poco::URI uri(url); + /// The query can carry a SAS token, which the error must not quote. + uri.setPathEtc(""); + if (!context->getRemoteHostFilter().checkURL( + uri, /* throw_on_error */ remote_host_filter_check == RemoteHostFilterCheck::Enforce)) + warnAboutDiskNotAllowedByRemoteHostFilter( + name, uri.getAuthority(), fmt::format("{}", uri.getHost()), context->getGlobalContext()); + }; + check_host(params.getConnectionURL()); + /// Writes to an ADLS Gen2 (OneLake) endpoint go to another host. + if (isAdlsGen2Endpoint(params.endpoint)) + check_host(getAdlsGen2Url(params.endpoint, params.auth_method)); + } + + const bool drop_credentials = params.checkOrDropServerManagedCredentials(); + if (drop_credentials) + LOG_WARNING( + getLogger("ObjectStorageFactory"), + "Loading the dynamic Azure disk {} with a client that has no credentials: it uses the server's own " + "identity, which is restricted for user queries (s3_allow_server_credentials_in_user_queries = 0). " + "The disk stays inaccessible; set s3_load_table_anonymously_if_credentials_restricted = 0 to fail " + "loading instead.", + name); + +>>>>>>> fee38d30170 (Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts) return std::make_shared( name, params.auth_method, AzureBlobStorage::getContainerClient(params, /*readonly=*/ false), std::move(azure_settings), diff --git a/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.cpp b/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.cpp index 6102225ee53cd..ee4b3a91b9fe7 100644 --- a/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.cpp @@ -13,6 +13,7 @@ #include #include +#include #include @@ -112,6 +113,14 @@ Azure::Storage::Files::DataLake::DataLakeFileClient makeAdlsGen2FileClient( auth_method); } +String getAdlsGen2Url(const AzureBlobStorage::Endpoint & endpoint, const AzureBlobStorage::AuthMethod & auth_method) +{ + const auto * connection_string = std::get_if(&auth_method); + if (connection_string && endpoint.sas_auth.empty() && endpoint.additional_params.empty()) + return Azure::Storage::_internal::ParseConnectionString(connection_string->toUnderType()).DataLakeServiceUrl.GetAbsoluteUrl(); + return buildAdlsGen2FileUrl(endpoint, /*blob_path=*/ ""); +} + WriteBufferFromAzureDataLakeStorage::WriteBufferFromAzureDataLakeStorage( const AzureBlobStorage::Endpoint & endpoint_, const AzureBlobStorage::AuthMethod & auth_method_, diff --git a/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.h b/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.h index be4acef948b28..1b7c2baf29719 100644 --- a/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureDataLakeStorage.h @@ -75,6 +75,9 @@ Azure::Storage::Files::DataLake::DataLakeFileClient makeAdlsGen2FileClient( bool isAdlsGen2Endpoint(const AzureBlobStorage::Endpoint & endpoint); +/// The URL of the service that the clients of `makeAdlsGen2FileClient` send requests to. +String getAdlsGen2Url(const AzureBlobStorage::Endpoint & endpoint, const AzureBlobStorage::AuthMethod & auth_method); + /// Build the ADLS Gen2 (DFS) write URL for a blob path. Exposed for unit testing. String buildAdlsGen2FileUrl(const AzureBlobStorage::Endpoint & endpoint, const String & blob_path); diff --git a/src/IO/AzureBlobStorage/PocoHTTPClient.cpp b/src/IO/AzureBlobStorage/PocoHTTPClient.cpp index ca63a35e32de6..575294ae1c562 100644 --- a/src/IO/AzureBlobStorage/PocoHTTPClient.cpp +++ b/src/IO/AzureBlobStorage/PocoHTTPClient.cpp @@ -370,6 +370,8 @@ std::unique_ptr PocoAzureHTTPClient::makeRequest /// validated at CREATE time (OneLake writes go to the `.dfs` host, reads to /// `.blob`). Normalize scheme and host as Poco::URI does at CREATE time (the /// scheme resolves the default port; the host is lower-cased) so both agree. + /// Disks are exempt; their redirects are still checked below. + if (!for_disk_azure) { Poco::URI request_uri; request_uri.setScheme(url.GetScheme()); diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h index b122defb4d7fe..2a11a431ff306 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -215,7 +215,9 @@ class DataLakeConfiguration : public BaseStorageConfiguration, public std::enabl void check(ContextPtr context) override { - if (ready_object_storage && ready_object_storage->getType() == ObjectStorageType::S3) + if (ready_object_storage + && (ready_object_storage->getType() == ObjectStorageType::S3 + || ready_object_storage->getType() == ObjectStorageType::Azure)) this->checkFormat(); else BaseStorageConfiguration::check(context); diff --git a/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml b/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml index 57a03003b8a5a..6d9fa832c7f9d 100644 --- a/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml +++ b/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml @@ -3,6 +3,7 @@ host:80 host:123 yandex.ru + onelake.blob.fabric.microsoft.com diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 23c28067f9de4..dceedae6b38fa 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -67,6 +67,9 @@ def test_config_with_hosts(start_cluster): "'http://127.0.0.1:1', 'container', 'data.csv', 'account', 'YQ==', 'CSV')", "INSERT INTO TABLE FUNCTION azureBlobStorage(" "'http://127.0.0.1:1', 'container', 'data.csv', 'account', 'YQ==', 'CSV', 'x UInt8') VALUES (1)", + "CREATE TABLE azure_blocked_disk (x UInt8) ENGINE = MergeTree ORDER BY x " + "SETTINGS disk = disk(type = azure_blob_storage, storage_account_url = 'http://127.0.0.1:1', " + "container_name = 'container', account_name = 'account', account_key = 'YQ==')", ], ) def test_azure_host_filter_before_client_creation(start_cluster, query): @@ -81,6 +84,82 @@ def test_azure_host_filter_before_client_creation(start_cluster, query): assert node5.query(container_requests) == before +<<<<<<< HEAD +======= +@pytest.mark.parametrize( + "endpoint, rejected_url", + [ + ( + "storage_account_url = 'https://onelake.blob.fabric.microsoft.com', " + "account_name = 'account', account_key = 'YQ=='", + "https://onelake.dfs.fabric.microsoft.com", + ), + ( + "connection_string = 'BlobEndpoint=https://onelake.blob.fabric.microsoft.com/;" + "DfsEndpoint=http://127.0.0.1:1;SharedAccessSignature=sv=2022&sig=c2FzY2FuYXJ5'", + "http://127.0.0.1:1", + ), + ( + "endpoint = 'BlobEndpoint=http://host:80/account/container', " + "connection_string = 'BlobEndpoint=http://127.0.0.1:1'", + "http://127.0.0.1:1", + ), + ], +) +def test_azure_disk_client_host(start_cluster, endpoint, rejected_url): + # Each disk names a host that node2 allows, but sends requests to one that node2 does not allow. + container_requests = ( + "SELECT sum(value) FROM system.events " + "WHERE event IN ('AzureGetProperties', 'AzureCreateContainer')" + ) + before = node2.query(container_requests) + error = node2.query_and_get_error( + "CREATE TABLE azure_client_host_disk (x UInt8) ENGINE = MergeTree ORDER BY x " + f"SETTINGS disk = disk(type = azure_blob_storage, {endpoint}, container_name = 'container')", + settings={"azure_sdk_max_retries": 0}, + ) + assert f'URL "{rejected_url}" is not allowed' in error + assert node2.query(container_requests) == before + + +def test_object_storage_queue_move_destination_host_filter(start_cluster): + # The source is on an allowed host (`host:80`), so only the move destination can be rejected. `node1` has no + # Keeper, so a rejection with "not allowed" must come from the settings validation, which runs before any + # Keeper access. + s3_queue = ( + "CREATE TABLE s3queue_move_host (x UInt8) " + "ENGINE = S3Queue('http://host:80/bucket/data/', 'key', 'secret', 'CSV') " + "SETTINGS mode = 'unordered', after_processing = 'move', " + "after_processing_move_uri = '{}', " + "after_processing_move_access_key_id = 'key', after_processing_move_secret_access_key = 'secret'" + ) + azure_queue = ( + "CREATE TABLE azurequeue_move_host (x UInt8) " + "ENGINE = AzureQueue('http://host:80', 'container', '*', 'account', 'YQ==', 'CSV') " + "SETTINGS mode = 'unordered', after_processing = 'move', " + "after_processing_move_connection_string = " + "'DefaultEndpointsProtocol=http;AccountName=account;AccountKey=YQ==;BlobEndpoint=http://host:123/account;', " + "after_processing_move_container = 'dst'" + ) + + try: + error = node1.query_and_get_error(s3_queue.format("http://host:123/bucket/")) + assert "not allowed" in error, error + + error = node1.query_and_get_error( + azure_queue, settings={"azure_sdk_max_retries": 0} + ) + assert "not allowed" in error, error + + # Control: an allowed destination passes the host check and only fails later, on the missing Keeper. + error = node1.query_and_get_error(s3_queue.format("http://host:80/bucket/")) + assert "not allowed" not in error, error + finally: + node1.query("DROP TABLE IF EXISTS s3queue_move_host SYNC") + node1.query("DROP TABLE IF EXISTS azurequeue_move_host SYNC") + + +>>>>>>> fee38d30170 (Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts) def test_config_with_only_primary_hosts(start_cluster): assert ( node2.query( diff --git a/tests/integration/test_merge_tree_azure_blob_storage/test.py b/tests/integration/test_merge_tree_azure_blob_storage/test.py index 87801c38f9b44..78bed0e5017b3 100644 --- a/tests/integration/test_merge_tree_azure_blob_storage/test.py +++ b/tests/integration/test_merge_tree_azure_blob_storage/test.py @@ -78,6 +78,7 @@ def cluster(): path, ], with_azurite=True, + stay_alive=True, ) logging.info("Starting cluster...") cluster.start() @@ -667,6 +668,93 @@ def test_endpoint_new_container(cluster): assert 10 == int(node.query("SELECT count() FROM test")) +<<<<<<< HEAD +======= +def test_endpoint_host_filter_on_restart(cluster): + # A query-created disk is checked against `remote_url_allow_hosts`. As for S3, a table loaded from + # existing metadata is kept with a warning when its host is no longer allowed; a new one is rejected. + node = cluster.instances[NODE_NAME] + port = cluster.azurite_port + allow_hosts_config = "/etc/clickhouse-server/config.d/remote_url_allow_hosts.xml" + + def azure_disk(container): + return f"""disk( + type = azure_blob_storage, + endpoint = 'http://azurite1:{port}/devstoreaccount1/{container}/data_prefix', + endpoint_contains_account_name = 'true', + account_name = 'devstoreaccount1', + account_key = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==')""" + + azure_query( + node, + f""" + DROP TABLE IF EXISTS test_host_filter SYNC; + + CREATE TABLE test_host_filter (a Int32) + ENGINE = MergeTree() ORDER BY tuple() + SETTINGS disk = {azure_disk('cont4')}; + + INSERT INTO test_host_filter SELECT number FROM numbers(10); + """, + ) + + try: + node.replace_config( + allow_hosts_config, + "blocked.invalid", + ) + node.restart_clickhouse() + assert 10 == int(node.query("SELECT count() FROM test_host_filter")) + assert node.contains_in_log("is not allowed by `remote_url_allow_hosts`") + assert 1 == int( + node.query( + f"SELECT count() FROM system.warnings WHERE message LIKE '%remote_url_allow_hosts%' AND message LIKE '%azurite1:{port}%'" + ) + ) + error = node.query_and_get_error( + f"CREATE TABLE test_host_filter_new (a Int32) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = {azure_disk('cont5')}" + ) + assert "UNACCEPTABLE_URL" in error, error + finally: + node.exec_in_container(["rm", "-f", allow_hosts_config]) + node.restart_clickhouse() + + assert 10 == int(node.query("SELECT count() FROM test_host_filter")) + node.query("DROP TABLE test_host_filter SYNC") + + +def test_reject_zero_min_upload_part_size_in_disk_config(cluster): + # Regression test for https://github.com/ClickHouse/ClickHouse/issues/81282: + # an Azure disk with min_upload_part_size = 0 used to reach BufferAllocationPolicy + # and trigger the internal `second_size > 0` assertion on write. + node = cluster.instances[NODE_NAME] + account_name = "devstoreaccount1" + container_name = "cont3" + data_prefix = "data_prefix" + port = cluster.azurite_port + + query = f""" + DROP TABLE IF EXISTS test_reject_zero_min_upload SYNC; + + CREATE TABLE test_reject_zero_min_upload (a Int32) + ENGINE = MergeTree() ORDER BY tuple() + SETTINGS disk = disk( + type = azure_blob_storage, + endpoint = 'http://azurite1:{port}/{account_name}/{container_name}/{data_prefix}', + endpoint_contains_account_name = 'true', + account_name = 'devstoreaccount1', + account_key = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', + container_already_exists = 1, + min_upload_part_size = 0, + skip_access_check = 0); + """ + + error = azure_query(node, query, expect_error=True) + assert "INVALID_SETTING_VALUE" in error, error + assert "azure_min_upload_part_size" in error, error + + +>>>>>>> fee38d30170 (Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts) def test_endpoint_without_prefix(cluster): node = cluster.instances[NODE_NAME] account_name = "devstoreaccount1" diff --git a/tests/integration/test_storage_iceberg_disks/test.py b/tests/integration/test_storage_iceberg_disks/test.py index 044881229f84a..a62d9bdd6336b 100644 --- a/tests/integration/test_storage_iceberg_disks/test.py +++ b/tests/integration/test_storage_iceberg_disks/test.py @@ -106,7 +106,7 @@ def generate_cluster_def(common_path, port, azure_container): disk_local_common,disk_s3_common,disk_azure_common,disk_s3_with_cache,disk_azure_with_cache - azurite1 + blocked.invalid """ From 041abda5bbcb174e066a6aba74bbf828cac36ef7 Mon Sep 17 00:00:00 2001 From: blau-ai <302218339+blau-ai@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:11:23 +0000 Subject: [PATCH 2/2] Resolve conflicts in backport of #124721 Kept the PR's Azure disk host-check, warning and test changes; dropped the unrelated master-only credential-restriction code and tests. --- .../AzureBlobStorageCommon.cpp | 74 ------------------- .../ObjectStorages/ObjectStorageFactory.cpp | 19 +---- .../test_allowed_url_from_config/test.py | 40 ---------- .../test.py | 34 --------- 4 files changed, 1 insertion(+), 166 deletions(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index ecd9590674610..ad9ea8bb940fb 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -235,80 +235,6 @@ String ConnectionParams::getConnectionURL() const return endpoint.storage_account_url; } -<<<<<<< HEAD -======= -static bool isServerManagedCredential(const AuthMethod & auth_method) -{ - return std::holds_alternative>(auth_method) - || std::holds_alternative>(auth_method); -} - -[[noreturn]] static void throwServerCredentialsRestricted() -{ - throw Exception( - ErrorCodes::ACCESS_DENIED, - "Azure access from user queries is not allowed to use the server's own identity (AKS workload identity " - "or the machine's managed identity). Provide explicit credentials: an `account_name` and `account_key`, " - "a connection string, or a SAS token" -#if !CLICKHOUSE_CLOUD - ", or enable the setting `s3_allow_server_credentials_in_user_queries`" -#endif - "."); -} - -bool ConnectionParams::usesServerManagedCredentials() const -{ - /// A request authorized by SAS does not use `auth_method`. - return endpoint.sas_auth.empty() && endpoint.additional_params.empty() && isServerManagedCredential(auth_method); -} - -bool ConnectionParams::checkOrDropServerManagedCredentials() const -{ - if (!forbid_implicit_credentials || !usesServerManagedCredentials()) - return false; - - if (anonymous_fallback_for_server_credentials) - return true; - - throwServerCredentialsRestricted(); -} - -void ConnectionParams::checkSessionMayUseServerIdentity(bool restrict_server_credentials) const -{ - if (restrict_server_credentials && follows_session_restriction && !forbid_implicit_credentials && usesServerManagedCredentials()) - throwServerCredentialsRestricted(); -} - -AuthMethod ConnectionParams::getPermittedAuthMethod() const -{ - if (checkOrDropServerManagedCredentials()) - throwServerCredentialsRestricted(); - return auth_method; -} - -String ConnectionParams::getConnectionURLWithoutQuery() const -{ - if (std::holds_alternative(auth_method)) - { - try - { - /// `ParseConnectionString` appends the SAS of the connection string to `BlobServiceUrl` as its query. - auto url = Azure::Storage::_internal::ParseConnectionString(getClientConnectionString(*this)).BlobServiceUrl; - const auto query = url.GetQueryParameters(); - for (const auto & [key, value] : query) - url.RemoveQueryParameter(key); - return url.GetAbsoluteUrl(); - } - catch (const std::logic_error & e) - { - translateAzureSdkParseError(e); - } - } - - return endpoint.storage_account_url.substr(0, endpoint.storage_account_url.find_first_of("?#")); -} - ->>>>>>> fee38d30170 (Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts) std::unique_ptr ConnectionParams::createForService() const { try diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp index 1b22bd6fa24a1..a1c63b02a460c 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp @@ -14,11 +14,7 @@ #if USE_AZURE_BLOB_STORAGE #include #include -<<<<<<< HEAD -======= #include -#include ->>>>>>> fee38d30170 (Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts) #endif #include @@ -249,7 +245,7 @@ static void registerAzureObjectStorage(ObjectStorageFactory & factory) const std::string & config_prefix, const ContextPtr & context, bool /* skip_access_check */, - RemoteHostFilterCheck /* remote_host_filter_check */) -> ObjectStoragePtr + RemoteHostFilterCheck remote_host_filter_check) -> ObjectStoragePtr { auto azure_settings = AzureBlobStorage::getRequestSettings(config, config_prefix, context->getSettingsRef()); @@ -266,8 +262,6 @@ static void registerAzureObjectStorage(ObjectStorageFactory & factory) .client_options = AzureBlobStorage::getClientOptions(context, context->getSettingsRef(), *azure_settings, /*for_disk=*/ true) }; -<<<<<<< HEAD -======= /// As for S3: a disk loaded from existing metadata is only reported, so that it fails neither /// the table load nor the server start. if (remote_host_filter_check != RemoteHostFilterCheck::None) @@ -288,17 +282,6 @@ static void registerAzureObjectStorage(ObjectStorageFactory & factory) check_host(getAdlsGen2Url(params.endpoint, params.auth_method)); } - const bool drop_credentials = params.checkOrDropServerManagedCredentials(); - if (drop_credentials) - LOG_WARNING( - getLogger("ObjectStorageFactory"), - "Loading the dynamic Azure disk {} with a client that has no credentials: it uses the server's own " - "identity, which is restricted for user queries (s3_allow_server_credentials_in_user_queries = 0). " - "The disk stays inaccessible; set s3_load_table_anonymously_if_credentials_restricted = 0 to fail " - "loading instead.", - name); - ->>>>>>> fee38d30170 (Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts) return std::make_shared( name, params.auth_method, AzureBlobStorage::getContainerClient(params, /*readonly=*/ false), std::move(azure_settings), diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index dceedae6b38fa..126d664e9c9ac 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -84,8 +84,6 @@ def test_azure_host_filter_before_client_creation(start_cluster, query): assert node5.query(container_requests) == before -<<<<<<< HEAD -======= @pytest.mark.parametrize( "endpoint, rejected_url", [ @@ -122,44 +120,6 @@ def test_azure_disk_client_host(start_cluster, endpoint, rejected_url): assert node2.query(container_requests) == before -def test_object_storage_queue_move_destination_host_filter(start_cluster): - # The source is on an allowed host (`host:80`), so only the move destination can be rejected. `node1` has no - # Keeper, so a rejection with "not allowed" must come from the settings validation, which runs before any - # Keeper access. - s3_queue = ( - "CREATE TABLE s3queue_move_host (x UInt8) " - "ENGINE = S3Queue('http://host:80/bucket/data/', 'key', 'secret', 'CSV') " - "SETTINGS mode = 'unordered', after_processing = 'move', " - "after_processing_move_uri = '{}', " - "after_processing_move_access_key_id = 'key', after_processing_move_secret_access_key = 'secret'" - ) - azure_queue = ( - "CREATE TABLE azurequeue_move_host (x UInt8) " - "ENGINE = AzureQueue('http://host:80', 'container', '*', 'account', 'YQ==', 'CSV') " - "SETTINGS mode = 'unordered', after_processing = 'move', " - "after_processing_move_connection_string = " - "'DefaultEndpointsProtocol=http;AccountName=account;AccountKey=YQ==;BlobEndpoint=http://host:123/account;', " - "after_processing_move_container = 'dst'" - ) - - try: - error = node1.query_and_get_error(s3_queue.format("http://host:123/bucket/")) - assert "not allowed" in error, error - - error = node1.query_and_get_error( - azure_queue, settings={"azure_sdk_max_retries": 0} - ) - assert "not allowed" in error, error - - # Control: an allowed destination passes the host check and only fails later, on the missing Keeper. - error = node1.query_and_get_error(s3_queue.format("http://host:80/bucket/")) - assert "not allowed" not in error, error - finally: - node1.query("DROP TABLE IF EXISTS s3queue_move_host SYNC") - node1.query("DROP TABLE IF EXISTS azurequeue_move_host SYNC") - - ->>>>>>> fee38d30170 (Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts) def test_config_with_only_primary_hosts(start_cluster): assert ( node2.query( diff --git a/tests/integration/test_merge_tree_azure_blob_storage/test.py b/tests/integration/test_merge_tree_azure_blob_storage/test.py index 78bed0e5017b3..0eeb2a0d34027 100644 --- a/tests/integration/test_merge_tree_azure_blob_storage/test.py +++ b/tests/integration/test_merge_tree_azure_blob_storage/test.py @@ -668,8 +668,6 @@ def test_endpoint_new_container(cluster): assert 10 == int(node.query("SELECT count() FROM test")) -<<<<<<< HEAD -======= def test_endpoint_host_filter_on_restart(cluster): # A query-created disk is checked against `remote_url_allow_hosts`. As for S3, a table loaded from # existing metadata is kept with a warning when its host is no longer allowed; a new one is rejected. @@ -723,38 +721,6 @@ def azure_disk(container): node.query("DROP TABLE test_host_filter SYNC") -def test_reject_zero_min_upload_part_size_in_disk_config(cluster): - # Regression test for https://github.com/ClickHouse/ClickHouse/issues/81282: - # an Azure disk with min_upload_part_size = 0 used to reach BufferAllocationPolicy - # and trigger the internal `second_size > 0` assertion on write. - node = cluster.instances[NODE_NAME] - account_name = "devstoreaccount1" - container_name = "cont3" - data_prefix = "data_prefix" - port = cluster.azurite_port - - query = f""" - DROP TABLE IF EXISTS test_reject_zero_min_upload SYNC; - - CREATE TABLE test_reject_zero_min_upload (a Int32) - ENGINE = MergeTree() ORDER BY tuple() - SETTINGS disk = disk( - type = azure_blob_storage, - endpoint = 'http://azurite1:{port}/{account_name}/{container_name}/{data_prefix}', - endpoint_contains_account_name = 'true', - account_name = 'devstoreaccount1', - account_key = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', - container_already_exists = 1, - min_upload_part_size = 0, - skip_access_check = 0); - """ - - error = azure_query(node, query, expect_error=True) - assert "INVALID_SETTING_VALUE" in error, error - assert "azure_min_upload_part_size" in error, error - - ->>>>>>> fee38d30170 (Merge pull request #124721 from groeneai/azure-disk-remote-url-allow-hosts) def test_endpoint_without_prefix(cluster): node = cluster.instances[NODE_NAME] account_name = "devstoreaccount1"