Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -210,13 +210,21 @@ String ContainerClientWrapper::GetBlobPath(const String & blob_name) const
"Failed to parse Azure connection string or blob URL: {}", e.what());
}

/// The connection string the clients are created from. A disk can set it together with an `endpoint`, whose host
/// the clients then do not use. The default-constructed `auth_method` of a SAS or catalog path is empty.
static const String & getClientConnectionString(const ConnectionParams & params)
{
const auto & connection_string = std::get<ConnectionString>(params.auth_method).toUnderType();
return connection_string.empty() ? params.endpoint.storage_account_url : connection_string;
}

String ConnectionParams::getConnectionURL() const
{
if (std::holds_alternative<ConnectionString>(auth_method))
{
try
{
auto parsed_connection_string = Azure::Storage::_internal::ParseConnectionString(endpoint.storage_account_url);
auto parsed_connection_string = Azure::Storage::_internal::ParseConnectionString(getClientConnectionString(*this));
return parsed_connection_string.BlobServiceUrl.GetAbsoluteUrl();
}
catch (const std::logic_error & e)
Expand Down Expand Up @@ -387,6 +395,8 @@ std::unique_ptr<ContainerClient> getContainerClient(const ConnectionParams & par
ProfileEvents::increment(ProfileEvents::DiskAzureCreateContainer);

auto raw_client = service_client->CreateBlobContainer(params.endpoint.container_name).Value;
/// `CreateBlobContainer` returns a client without `ClickhouseOptions`.
raw_client.SetClickhouseOptions(params.client_options.ClickhouseOptions);
return std::make_unique<ContainerClient>(std::move(raw_client), params.endpoint.prefix);
}
catch (const Azure::Storage::StorageException & e)
Expand Down
82 changes: 56 additions & 26 deletions src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageFactory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#if USE_AZURE_BLOB_STORAGE
#include <Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
#include <Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#include <Disks/IO/WriteBufferFromAzureDataLakeStorage.h>
#endif

#include <Disks/DiskObjectStorage/ObjectStorages/Web/WebObjectStorage.h>
Expand Down Expand Up @@ -102,6 +103,32 @@ ObjectStoragePtr ObjectStorageFactory::create(
return it->second(name, config, config_prefix, context, skip_access_check, remote_host_filter_check);
}

/// A custom disk with an endpoint that remote_url_allow_hosts does not allow: a log line per disk
/// and one system.warnings entry that lists all such disks. `endpoint` is what `allow_list_entry`
/// (an <s3_bucket> or <host> element) has to allow.
[[maybe_unused]] static void warnAboutDiskNotAllowedByRemoteHostFilter(
const std::string & disk_name, const std::string & endpoint, const std::string & allow_list_entry, const ContextPtr & global_context)
{
LOG_WARNING(
getLogger("ObjectStorageFactory"),
"Custom disk `{}` uses endpoint `{}` that is not allowed by `remote_url_allow_hosts`. The disk was created before "
"this check and is kept while loading existing metadata (server start or UNDROP). Add `{}` "
"to the list or recreate the tables that use it.",
disk_name, endpoint, allow_list_entry);

/// system.warnings keeps one message per type, so the message lists all disks seen so far.
/// No lock: disk creation runs under storage_policies_mutex in Context::getOrCreateDisk.
static std::set<std::string> disks;
disks.insert(fmt::format("`{}` -> `{}`", disk_name, endpoint));
global_context->addOrUpdateWarningMessage(
Context::WarningType::CUSTOM_DISK_NOT_ALLOWED_BY_REMOTE_URL_ALLOW_HOSTS,
PreformattedMessage::create(
"Custom disks with endpoints that are not allowed by `remote_url_allow_hosts` were kept while loading existing "
"metadata (server start or UNDROP): {}. Add `<s3_bucket>host/bucket</s3_bucket>` (S3) or `<host>host</host>` "
"(Azure) entries for them or recreate the tables that use them.",
fmt::join(disks, ", ")));
}

#if USE_AWS_S3
namespace
{
Expand Down Expand Up @@ -132,35 +159,14 @@ static std::string getEndpoint(
return context->getMacros()->expand(config.getString(config_prefix + ".endpoint"));
}

/// A custom disk with an endpoint that remote_url_allow_hosts does not allow: a log line per disk
/// and one system.warnings entry that lists all such disks.
static void warnAboutDiskNotAllowedByRemoteHostFilter(const std::string & disk_name, const S3::URI & uri, const ContextPtr & global_context)
/// host[:port]/bucket of an S3 endpoint, as in an <s3_bucket> entry.
static std::string getS3BucketEntry(const S3::URI & uri)
{
Poco::URI endpoint_uri(uri.endpoint);
/// host[:port], as in an <s3_bucket> entry.
std::string host_and_port = endpoint_uri.getAuthority();
if (auto at = host_and_port.rfind('@'); at != std::string::npos)
host_and_port.erase(0, at + 1);
const std::string s3_bucket_entry = host_and_port + "/" + uri.bucket;

LOG_WARNING(
getLogger("ObjectStorageFactory"),
"Custom disk `{}` uses endpoint `{}` that is not allowed by `remote_url_allow_hosts`. The disk was created before "
"this check and is kept while loading existing metadata (server start or UNDROP). Add `<s3_bucket>{}</s3_bucket>` "
"to the list or recreate the tables that use it.",
disk_name, s3_bucket_entry, s3_bucket_entry);

/// system.warnings keeps one message per type, so the message lists all disks seen so far.
/// No lock: disk creation runs under storage_policies_mutex in Context::getOrCreateDisk.
static std::set<std::string> disks;
disks.insert(fmt::format("`{}` -> `{}`", disk_name, s3_bucket_entry));
global_context->addOrUpdateWarningMessage(
Context::WarningType::CUSTOM_DISK_NOT_ALLOWED_BY_REMOTE_URL_ALLOW_HOSTS,
PreformattedMessage::create(
"Custom disks with endpoints that are not allowed by `remote_url_allow_hosts` were kept while loading existing "
"metadata (server start or UNDROP): {}. Add `<s3_bucket>host/bucket</s3_bucket>` entries for them or recreate "
"the tables that use them.",
fmt::join(disks, ", ")));
return host_and_port + "/" + uri.bucket;
}

static void registerS3ObjectStorage(ObjectStorageFactory & factory)
Expand All @@ -186,7 +192,11 @@ static void registerS3ObjectStorage(ObjectStorageFactory & factory)
&& !uri.checkRemoteHostFilter(
context->getGlobalContext()->getRemoteHostFilter(),
/* throw_on_error */ remote_host_filter_check == RemoteHostFilterCheck::Enforce))
warnAboutDiskNotAllowedByRemoteHostFilter(name, uri, context->getGlobalContext());
{
const auto s3_bucket_entry = getS3BucketEntry(uri);
warnAboutDiskNotAllowedByRemoteHostFilter(
name, s3_bucket_entry, fmt::format("<s3_bucket>{}</s3_bucket>", s3_bucket_entry), context->getGlobalContext());
}
auto client = getClient(endpoint, *settings, context, /* for_disk_s3 */ true, name);
auto key_generator = getKeyGenerator(uri, config, config_prefix);

Expand Down Expand Up @@ -235,7 +245,7 @@ static void registerAzureObjectStorage(ObjectStorageFactory & factory)
const std::string & config_prefix,
const ContextPtr & context,
bool /* skip_access_check */,
RemoteHostFilterCheck /* remote_host_filter_check */) -> ObjectStoragePtr
RemoteHostFilterCheck remote_host_filter_check) -> ObjectStoragePtr
{
auto azure_settings = AzureBlobStorage::getRequestSettings(config, config_prefix, context->getSettingsRef());

Expand All @@ -252,6 +262,26 @@ static void registerAzureObjectStorage(ObjectStorageFactory & factory)
.client_options = AzureBlobStorage::getClientOptions(context, context->getSettingsRef(), *azure_settings, /*for_disk=*/ true)
};

/// As for S3: a disk loaded from existing metadata is only reported, so that it fails neither
/// the table load nor the server start.
if (remote_host_filter_check != RemoteHostFilterCheck::None)
{
auto check_host = [&](const String & url)
{
Poco::URI uri(url);
/// The query can carry a SAS token, which the error must not quote.
uri.setPathEtc("");
if (!context->getRemoteHostFilter().checkURL(
uri, /* throw_on_error */ remote_host_filter_check == RemoteHostFilterCheck::Enforce))
warnAboutDiskNotAllowedByRemoteHostFilter(
name, uri.getAuthority(), fmt::format("<host>{}</host>", uri.getHost()), context->getGlobalContext());
};
check_host(params.getConnectionURL());
/// Writes to an ADLS Gen2 (OneLake) endpoint go to another host.
if (isAdlsGen2Endpoint(params.endpoint))
check_host(getAdlsGen2Url(params.endpoint, params.auth_method));
}

return std::make_shared<AzureObjectStorage>(
name,
params.auth_method, AzureBlobStorage::getContainerClient(params, /*readonly=*/ false), std::move(azure_settings),
Expand Down
9 changes: 9 additions & 0 deletions src/Disks/IO/WriteBufferFromAzureDataLakeStorage.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <Common/Stopwatch.h>

#include <azure/core/io/body_stream.hpp>
#include <azure/storage/common/storage_credential.hpp>
#include <azure/storage/files/datalake/datalake_options.hpp>


Expand Down Expand Up @@ -112,6 +113,14 @@ Azure::Storage::Files::DataLake::DataLakeFileClient makeAdlsGen2FileClient(
auth_method);
}

String getAdlsGen2Url(const AzureBlobStorage::Endpoint & endpoint, const AzureBlobStorage::AuthMethod & auth_method)
{
const auto * connection_string = std::get_if<AzureBlobStorage::ConnectionString>(&auth_method);
if (connection_string && endpoint.sas_auth.empty() && endpoint.additional_params.empty())
return Azure::Storage::_internal::ParseConnectionString(connection_string->toUnderType()).DataLakeServiceUrl.GetAbsoluteUrl();
return buildAdlsGen2FileUrl(endpoint, /*blob_path=*/ "");
}

WriteBufferFromAzureDataLakeStorage::WriteBufferFromAzureDataLakeStorage(
const AzureBlobStorage::Endpoint & endpoint_,
const AzureBlobStorage::AuthMethod & auth_method_,
Expand Down
3 changes: 3 additions & 0 deletions src/Disks/IO/WriteBufferFromAzureDataLakeStorage.h
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ Azure::Storage::Files::DataLake::DataLakeFileClient makeAdlsGen2FileClient(

bool isAdlsGen2Endpoint(const AzureBlobStorage::Endpoint & endpoint);

/// The URL of the service that the clients of `makeAdlsGen2FileClient` send requests to.
String getAdlsGen2Url(const AzureBlobStorage::Endpoint & endpoint, const AzureBlobStorage::AuthMethod & auth_method);

/// Build the ADLS Gen2 (DFS) write URL for a blob path. Exposed for unit testing.
String buildAdlsGen2FileUrl(const AzureBlobStorage::Endpoint & endpoint, const String & blob_path);

Expand Down
2 changes: 2 additions & 0 deletions src/IO/AzureBlobStorage/PocoHTTPClient.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,8 @@ std::unique_ptr<Azure::Core::Http::RawResponse> PocoAzureHTTPClient::makeRequest
/// validated at CREATE time (OneLake writes go to the `.dfs` host, reads to
/// `.blob`). Normalize scheme and host as Poco::URI does at CREATE time (the
/// scheme resolves the default port; the host is lower-cased) so both agree.
/// Disks are exempt; their redirects are still checked below.
if (!for_disk_azure)
{
Poco::URI request_uri;
request_uri.setScheme(url.GetScheme());
Expand Down
4 changes: 3 additions & 1 deletion src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,9 @@ class DataLakeConfiguration : public BaseStorageConfiguration, public std::enabl

void check(ContextPtr context) override
{
if (ready_object_storage && ready_object_storage->getType() == ObjectStorageType::S3)
if (ready_object_storage
&& (ready_object_storage->getType() == ObjectStorageType::S3
|| ready_object_storage->getType() == ObjectStorageType::Azure))
this->checkFormat();
else
BaseStorageConfiguration::check(context);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
<host>host:80</host>
<host>host:123</host>
<host>yandex.ru</host>
<host>onelake.blob.fabric.microsoft.com</host>
</remote_url_allow_hosts>
</clickhouse>

39 changes: 39 additions & 0 deletions tests/integration/test_allowed_url_from_config/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ def test_config_with_hosts(start_cluster):
"'http://127.0.0.1:1', 'container', 'data.csv', 'account', 'YQ==', 'CSV')",
"INSERT INTO TABLE FUNCTION azureBlobStorage("
"'http://127.0.0.1:1', 'container', 'data.csv', 'account', 'YQ==', 'CSV', 'x UInt8') VALUES (1)",
"CREATE TABLE azure_blocked_disk (x UInt8) ENGINE = MergeTree ORDER BY x "
"SETTINGS disk = disk(type = azure_blob_storage, storage_account_url = 'http://127.0.0.1:1', "
"container_name = 'container', account_name = 'account', account_key = 'YQ==')",
],
)
def test_azure_host_filter_before_client_creation(start_cluster, query):
Expand All @@ -81,6 +84,42 @@ def test_azure_host_filter_before_client_creation(start_cluster, query):
assert node5.query(container_requests) == before


@pytest.mark.parametrize(
"endpoint, rejected_url",
[
(
"storage_account_url = 'https://onelake.blob.fabric.microsoft.com', "
"account_name = 'account', account_key = 'YQ=='",
"https://onelake.dfs.fabric.microsoft.com",
),
(
"connection_string = 'BlobEndpoint=https://onelake.blob.fabric.microsoft.com/;"
"DfsEndpoint=http://127.0.0.1:1;SharedAccessSignature=sv=2022&sig=c2FzY2FuYXJ5'",
"http://127.0.0.1:1",
),
(
"endpoint = 'BlobEndpoint=http://host:80/account/container', "
"connection_string = 'BlobEndpoint=http://127.0.0.1:1'",
"http://127.0.0.1:1",
),
],
)
def test_azure_disk_client_host(start_cluster, endpoint, rejected_url):
# Each disk names a host that node2 allows, but sends requests to one that node2 does not allow.
container_requests = (
"SELECT sum(value) FROM system.events "
"WHERE event IN ('AzureGetProperties', 'AzureCreateContainer')"
)
before = node2.query(container_requests)
error = node2.query_and_get_error(
"CREATE TABLE azure_client_host_disk (x UInt8) ENGINE = MergeTree ORDER BY x "
f"SETTINGS disk = disk(type = azure_blob_storage, {endpoint}, container_name = 'container')",
settings={"azure_sdk_max_retries": 0},
)
assert f'URL "{rejected_url}" is not allowed' in error
assert node2.query(container_requests) == before


def test_object_storage_queue_move_destination_host_filter(start_cluster):
# The source is on an allowed host (`host:80`), so only the move destination can be rejected. `node1` has no
# Keeper, so a rejection with "not allowed" must come from the settings validation, which runs before any
Expand Down
54 changes: 54 additions & 0 deletions tests/integration/test_merge_tree_azure_blob_storage/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ def cluster():
path,
],
with_azurite=True,
stay_alive=True,
)
logging.info("Starting cluster...")
cluster.start()
Expand Down Expand Up @@ -667,6 +668,59 @@ def test_endpoint_new_container(cluster):
assert 10 == int(node.query("SELECT count() FROM test"))


def test_endpoint_host_filter_on_restart(cluster):
# A query-created disk is checked against `remote_url_allow_hosts`. As for S3, a table loaded from
# existing metadata is kept with a warning when its host is no longer allowed; a new one is rejected.
node = cluster.instances[NODE_NAME]
port = cluster.azurite_port
allow_hosts_config = "/etc/clickhouse-server/config.d/remote_url_allow_hosts.xml"

def azure_disk(container):
return f"""disk(
type = azure_blob_storage,
endpoint = 'http://azurite1:{port}/devstoreaccount1/{container}/data_prefix',
endpoint_contains_account_name = 'true',
account_name = 'devstoreaccount1',
account_key = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==')"""

azure_query(
node,
f"""
DROP TABLE IF EXISTS test_host_filter SYNC;

CREATE TABLE test_host_filter (a Int32)
ENGINE = MergeTree() ORDER BY tuple()
SETTINGS disk = {azure_disk('cont4')};

INSERT INTO test_host_filter SELECT number FROM numbers(10);
""",
)

try:
node.replace_config(
allow_hosts_config,
"<clickhouse><remote_url_allow_hosts><host>blocked.invalid</host></remote_url_allow_hosts></clickhouse>",
)
node.restart_clickhouse()
assert 10 == int(node.query("SELECT count() FROM test_host_filter"))
assert node.contains_in_log("is not allowed by `remote_url_allow_hosts`")
assert 1 == int(
node.query(
f"SELECT count() FROM system.warnings WHERE message LIKE '%remote_url_allow_hosts%' AND message LIKE '%azurite1:{port}%'"
)
)
error = node.query_and_get_error(
f"CREATE TABLE test_host_filter_new (a Int32) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = {azure_disk('cont5')}"
)
assert "UNACCEPTABLE_URL" in error, error
finally:
node.exec_in_container(["rm", "-f", allow_hosts_config])
node.restart_clickhouse()

assert 10 == int(node.query("SELECT count() FROM test_host_filter"))
node.query("DROP TABLE test_host_filter SYNC")


def test_endpoint_without_prefix(cluster):
node = cluster.instances[NODE_NAME]
account_name = "devstoreaccount1"
Expand Down
2 changes: 1 addition & 1 deletion tests/integration/test_storage_iceberg_disks/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ def generate_cluster_def(common_path, port, azure_container):
</storage_configuration>
<allowed_disks_for_table_engines>disk_local_common,disk_s3_common,disk_azure_common,disk_s3_with_cache,disk_azure_with_cache</allowed_disks_for_table_engines>
<remote_url_allow_hosts>
<host>azurite1</host>
<host>blocked.invalid</host>
</remote_url_allow_hosts>
</clickhouse>
"""
Expand Down
Loading