From f913b454fffcfd3d9afd7ba25883eb5602da83a5 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Wed, 12 Mar 2025 21:03:19 +0000 Subject: [PATCH 1/4] samples(storagecontrol): add samples for anywhere cache --- .../storage_control_anywhere_cache_samples.cc | 227 ++++++++++++++++++ .../samples/storage_control_folder_samples.cc | 2 +- 2 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 google/cloud/storagecontrol/v2/samples/storage_control_anywhere_cache_samples.cc diff --git a/google/cloud/storagecontrol/v2/samples/storage_control_anywhere_cache_samples.cc b/google/cloud/storagecontrol/v2/samples/storage_control_anywhere_cache_samples.cc new file mode 100644 index 0000000000000..590ac9f931186 --- /dev/null +++ b/google/cloud/storagecontrol/v2/samples/storage_control_anywhere_cache_samples.cc @@ -0,0 +1,227 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "google/cloud/storagecontrol/v2/storage_control_client.h" +#include "google/cloud/internal/getenv.h" +#include "google/cloud/testing_util/example_driver.h" +#include +#include +#include +#include + +namespace { + +void CreateAnywhereCache( + google::cloud::storagecontrol_v2::StorageControlClient client, + std::vector const& argv) { + // [START storage_control_create_anywhere_cache] + namespace storagecontrol = google::cloud::storagecontrol_v2; + [](storagecontrol::StorageControlClient client, std::string bucket_name, + std::string cache_name, std::string zone_name) { + google::storage::control::v2::AnywhereCache cache; + cache.set_name(cache_name); + cache.set_zone(zone_name); + + google::storage::control::v2::CreateAnywhereCacheRequest request; + request.set_parent(std::string{"projects/_/buckets/"} + bucket_name); + *request.mutable_anywhere_cache() = cache; + + // Start a create operation and block until it completes. Real applications + // may want to setup a callback, wait on a coroutine, or poll until it + // completes. + auto anywhere_cache = client.CreateAnywhereCache(request).get(); + if (!anywhere_cache) throw std::move(anywhere_cache).status(); + std::cout << "Created anywhere cache: " << anywhere_cache->name() << "\n"; + } + // [END storage_control_create_anywhere_cache] + (std::move(client), argv.at(0), argv.at(1), argv.at(2)); +} + +void GetAnywhereCache( + google::cloud::storagecontrol_v2::StorageControlClient client, + std::vector const& argv) { + // [START storage_control_get_anywhere_cache] + namespace storagecontrol = google::cloud::storagecontrol_v2; + [](storagecontrol::StorageControlClient client, std::string cache_name) { + auto anywhere_cache = client.GetAnywhereCache(cache_name); + if (!anywhere_cache) throw std::move(anywhere_cache).status(); + std::cout << "Got anywhere cache: " << anywhere_cache->name() << "\n"; + } + // [END storage_control_get_anywhere_cache] + (std::move(client), argv.at(0)); +} + +void ListAnywhereCaches( + google::cloud::storagecontrol_v2::StorageControlClient client, + std::vector const& argv) { + // [START storage_control_list_anywhere_caches] + namespace storagecontrol = google::cloud::storagecontrol_v2; + [](storagecontrol::StorageControlClient client, std::string bucket_name) { + auto const parent = std::string{"projects/_/buckets/"} + bucket_name; + for (auto anywhere_cache : client.ListAnywhereCaches(parent)) { + if (!anywhere_cache) throw std::move(anywhere_cache).status(); + std::cout << anywhere_cache->name() << "\n"; + } + } + // [END storage_control_list_anywhere_caches] + (std::move(client), argv.at(0)); +} + +void UpdateAnywhereCache( + google::cloud::storagecontrol_v2::StorageControlClient client, + std::vector const& argv) { + // [START storage_control_update_anywhere_cache] + namespace storagecontrol = google::cloud::storagecontrol_v2; + [](storagecontrol::StorageControlClient client, std::string cache_name, + std::string admission_policy) { + google::storage::control::v2::AnywhereCache cache; + google::protobuf::FieldMask field_mask; + field_mask.add_paths("admission_policy"); + cache.set_name(cache_name); + cache.set_admission_policy(admission_policy); + // Start an update operation and block until it completes. Real applications + // may want to setup a callback, wait on a coroutine, or poll until it + // completes. + auto anywhere_cache = client.UpdateAnywhereCache(cache, field_mask).get(); + if (!anywhere_cache) throw std::move(anywhere_cache).status(); + std::cout << "Updated anywhere cache: " << anywhere_cache->name() << "\n"; + } + // [END storage_control_update_anywhere_cache] + (std::move(client), argv.at(0), argv.at(1)); +} + +void PauseAnywhereCache( + google::cloud::storagecontrol_v2::StorageControlClient client, + std::vector const& argv) { + // [START storage_control_pause_anywhere_cache] + namespace storagecontrol = google::cloud::storagecontrol_v2; + [](storagecontrol::StorageControlClient client, std::string cache_name) { + auto anywhere_cache = client.PauseAnywhereCache(cache_name); + if (!anywhere_cache) throw std::move(anywhere_cache).status(); + std::cout << "Paused anywhere cache: " << anywhere_cache->name() << "\n"; + } + // [END storage_control_pause_anywhere_cache] + (std::move(client), argv.at(0)); +} + +void ResumeAnywhereCache( + google::cloud::storagecontrol_v2::StorageControlClient client, + std::vector const& argv) { + // [START storage_control_resume_anywhere_cache] + namespace storagecontrol = google::cloud::storagecontrol_v2; + [](storagecontrol::StorageControlClient client, std::string cache_name) { + auto anywhere_cache = client.ResumeAnywhereCache(cache_name); + if (!anywhere_cache) throw std::move(anywhere_cache).status(); + std::cout << "Resumed anywhere cache: " << anywhere_cache->name() << "\n"; + } + // [END storage_control_resume_anywhere_cache] + (std::move(client), argv.at(0)); +} + +void DisableAnywhereCache( + google::cloud::storagecontrol_v2::StorageControlClient client, + std::vector const& argv) { + // [START storage_control_disable_anywhere_cache] + namespace storagecontrol = google::cloud::storagecontrol_v2; + [](storagecontrol::StorageControlClient client, std::string cache_name) { + auto anywhere_cache = client.DisableAnywhereCache(cache_name); + if (!anywhere_cache) throw std::move(anywhere_cache).status(); + std::cout << "Disabled anywhere cache: " << anywhere_cache->name() << "\n"; + } + // [END storage_control_disable_anywhere_cache] + (std::move(client), argv.at(0)); +} + +void AutoRun(std::vector const& argv) { + namespace examples = google::cloud::testing_util; + namespace storagecontrol = google::cloud::storagecontrol_v2; + if (!argv.empty()) throw examples::Usage{"auto"}; + examples::CheckEnvironmentVariablesAreSet( + {"GOOGLE_CLOUD_CPP_STORAGE_TEST_BUCKET_NAME", + "GOOGLE_CLOUD_CPP_TEST_ZONE"}); + auto const bucket_name = google::cloud::internal::GetEnv( + "GOOGLE_CLOUD_CPP_STORAGE_TEST_BUCKET_NAME") + .value(); + auto const zone_name = + google::cloud::internal::GetEnv("GOOGLE_CLOUD_CPP_TEST_ZONE").value(); + auto client = storagecontrol::StorageControlClient( + storagecontrol::MakeStorageControlConnection()); + + auto const cache_name = + "projects/_/buckets/" + bucket_name + "/anywhereCaches/" + zone_name; + + std::cout << "\nRunning CreateAnywhereCache() example" << std::endl; + CreateAnywhereCache(client, {bucket_name, cache_name, zone_name}); + + std::cout << "\nRunning GetAnywhereCache() example" << std::endl; + GetAnywhereCache(client, {cache_name}); + + std::cout << "\nRunning ListAnywhereCaches() example" << std::endl; + ListAnywhereCaches(client, {bucket_name}); + + std::cout << "\nRunning UpdateAnywhereCache() example" << std::endl; + UpdateAnywhereCache(client, {cache_name, "admit-on-second-miss"}); + + std::cout << "\nRunning PauseAnywhereCache() example" << std::endl; + PauseAnywhereCache(client, {cache_name}); + + std::cout << "\nRunning ResumeAnywhereCache() example" << std::endl; + ResumeAnywhereCache(client, {cache_name}); + + std::cout << "\nRunning DisableAnywhereCache() example" << std::endl; + DisableAnywhereCache(client, {cache_name}); +} + +} // namespace + +int main(int argc, char* argv[]) { // NOLINT(bugprone-exception-escape) + using google::cloud::testing_util::Example; + namespace storagecontrol = google::cloud::storagecontrol_v2; + using ClientCommand = std::function argv)>; + + auto make_entry = [](std::string name, + std::vector const& arg_names, + ClientCommand const& command) { + auto adapter = [=](std::vector argv) { + if ((argv.size() == 1 && argv[0] == "--help") || + argv.size() != arg_names.size()) { + std::string usage = name; + for (auto const& a : arg_names) usage += " <" + a + ">"; + throw google::cloud::testing_util::Usage{std::move(usage)}; + } + auto client = storagecontrol::StorageControlClient( + storagecontrol::MakeStorageControlConnection()); + command(client, std::move(argv)); + }; + return google::cloud::testing_util::Commands::value_type(std::move(name), + adapter); + }; + + Example example({ + make_entry("create-anywhere-cache", + {"bucket-name", "cache-name", "zone-name"}, + CreateAnywhereCache), + make_entry("get-anywhere-cache", {"cache-name"}, GetAnywhereCache), + make_entry("list-anywhere-caches", {"bucket-name"}, ListAnywhereCaches), + make_entry("update-anywhere-cache", {"cache-name", "admission-policy"}, + UpdateAnywhereCache), + make_entry("pause-anywhere-cache", {"cache-name"}, PauseAnywhereCache), + make_entry("resume-anywhere-cache", {"cache-name"}, ResumeAnywhereCache), + make_entry("disable-anywhere-cache", {"cache-name"}, + DisableAnywhereCache), + {"auto", AutoRun}, + }); + return example.Run(argc, argv); +} diff --git a/google/cloud/storagecontrol/v2/samples/storage_control_folder_samples.cc b/google/cloud/storagecontrol/v2/samples/storage_control_folder_samples.cc index 12bedbae3d64d..5f0ebe81f0ab0 100644 --- a/google/cloud/storagecontrol/v2/samples/storage_control_folder_samples.cc +++ b/google/cloud/storagecontrol/v2/samples/storage_control_folder_samples.cc @@ -158,7 +158,7 @@ void AutoRun(std::vector const& argv) { std::chrono::system_clock::now() - std::chrono::hours(48); // This is the only example that cleans up stale folders. The examples run in // parallel (within a build and across the builds), having multiple examples - // doing the same cleanup is probably more trouble than it is worth. + // doing the same cleanup is probably more gl than it is worth. std::cout << "\nRemoving stale folders for examples" << std::endl; RemoveStaleFolders(client, bucket_name, prefix, create_time_limit); From 1517200472db849b8eec2a5938eb77c1dc7d07e5 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 8 May 2025 16:47:02 +0000 Subject: [PATCH 2/4] skip anywhere cache samples tests --- ci/cloudbuild/builds/integration-production.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/cloudbuild/builds/integration-production.sh b/ci/cloudbuild/builds/integration-production.sh index fb92d5665573c..66ca4496d87fa 100755 --- a/ci/cloudbuild/builds/integration-production.sh +++ b/ci/cloudbuild/builds/integration-production.sh @@ -35,6 +35,8 @@ excluded_rules=( # This sample uses HMAC keys, which are very limited in production (at most # 5 per service account). Disabled for now. "-//google/cloud/storage/examples:storage_service_account_samples" + # This sample can be very long running due to creation time of AnywhereCache + "-//google/cloud/storagecontrol:v2_samples_storage_control_anywhere_cache_samples" ) io::log_h2 "Running the integration tests against prod" From 0c8229a31301dde214ad4f107ce5b67f2a45074b Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 8 May 2025 16:51:49 +0000 Subject: [PATCH 3/4] fix typo --- .../storagecontrol/v2/samples/storage_control_folder_samples.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/storagecontrol/v2/samples/storage_control_folder_samples.cc b/google/cloud/storagecontrol/v2/samples/storage_control_folder_samples.cc index 5f0ebe81f0ab0..12bedbae3d64d 100644 --- a/google/cloud/storagecontrol/v2/samples/storage_control_folder_samples.cc +++ b/google/cloud/storagecontrol/v2/samples/storage_control_folder_samples.cc @@ -158,7 +158,7 @@ void AutoRun(std::vector const& argv) { std::chrono::system_clock::now() - std::chrono::hours(48); // This is the only example that cleans up stale folders. The examples run in // parallel (within a build and across the builds), having multiple examples - // doing the same cleanup is probably more gl than it is worth. + // doing the same cleanup is probably more trouble than it is worth. std::cout << "\nRemoving stale folders for examples" << std::endl; RemoveStaleFolders(client, bucket_name, prefix, create_time_limit); From f2fefd397d853b6d917c4b258ad321d20edfe553 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 8 May 2025 17:39:01 +0000 Subject: [PATCH 4/4] avoid copies --- .../storage_control_anywhere_cache_samples.cc | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/google/cloud/storagecontrol/v2/samples/storage_control_anywhere_cache_samples.cc b/google/cloud/storagecontrol/v2/samples/storage_control_anywhere_cache_samples.cc index 590ac9f931186..e8ca0dabca6af 100644 --- a/google/cloud/storagecontrol/v2/samples/storage_control_anywhere_cache_samples.cc +++ b/google/cloud/storagecontrol/v2/samples/storage_control_anywhere_cache_samples.cc @@ -27,8 +27,9 @@ void CreateAnywhereCache( std::vector const& argv) { // [START storage_control_create_anywhere_cache] namespace storagecontrol = google::cloud::storagecontrol_v2; - [](storagecontrol::StorageControlClient client, std::string bucket_name, - std::string cache_name, std::string zone_name) { + [](storagecontrol::StorageControlClient client, + std::string const& bucket_name, std::string const& cache_name, + std::string const& zone_name) { google::storage::control::v2::AnywhereCache cache; cache.set_name(cache_name); cache.set_zone(zone_name); @@ -53,7 +54,8 @@ void GetAnywhereCache( std::vector const& argv) { // [START storage_control_get_anywhere_cache] namespace storagecontrol = google::cloud::storagecontrol_v2; - [](storagecontrol::StorageControlClient client, std::string cache_name) { + [](storagecontrol::StorageControlClient client, + std::string const& cache_name) { auto anywhere_cache = client.GetAnywhereCache(cache_name); if (!anywhere_cache) throw std::move(anywhere_cache).status(); std::cout << "Got anywhere cache: " << anywhere_cache->name() << "\n"; @@ -67,7 +69,8 @@ void ListAnywhereCaches( std::vector const& argv) { // [START storage_control_list_anywhere_caches] namespace storagecontrol = google::cloud::storagecontrol_v2; - [](storagecontrol::StorageControlClient client, std::string bucket_name) { + [](storagecontrol::StorageControlClient client, + std::string const& bucket_name) { auto const parent = std::string{"projects/_/buckets/"} + bucket_name; for (auto anywhere_cache : client.ListAnywhereCaches(parent)) { if (!anywhere_cache) throw std::move(anywhere_cache).status(); @@ -83,8 +86,8 @@ void UpdateAnywhereCache( std::vector const& argv) { // [START storage_control_update_anywhere_cache] namespace storagecontrol = google::cloud::storagecontrol_v2; - [](storagecontrol::StorageControlClient client, std::string cache_name, - std::string admission_policy) { + [](storagecontrol::StorageControlClient client, std::string const& cache_name, + std::string const& admission_policy) { google::storage::control::v2::AnywhereCache cache; google::protobuf::FieldMask field_mask; field_mask.add_paths("admission_policy"); @@ -106,7 +109,8 @@ void PauseAnywhereCache( std::vector const& argv) { // [START storage_control_pause_anywhere_cache] namespace storagecontrol = google::cloud::storagecontrol_v2; - [](storagecontrol::StorageControlClient client, std::string cache_name) { + [](storagecontrol::StorageControlClient client, + std::string const& cache_name) { auto anywhere_cache = client.PauseAnywhereCache(cache_name); if (!anywhere_cache) throw std::move(anywhere_cache).status(); std::cout << "Paused anywhere cache: " << anywhere_cache->name() << "\n"; @@ -120,7 +124,8 @@ void ResumeAnywhereCache( std::vector const& argv) { // [START storage_control_resume_anywhere_cache] namespace storagecontrol = google::cloud::storagecontrol_v2; - [](storagecontrol::StorageControlClient client, std::string cache_name) { + [](storagecontrol::StorageControlClient client, + std::string const& cache_name) { auto anywhere_cache = client.ResumeAnywhereCache(cache_name); if (!anywhere_cache) throw std::move(anywhere_cache).status(); std::cout << "Resumed anywhere cache: " << anywhere_cache->name() << "\n"; @@ -134,7 +139,8 @@ void DisableAnywhereCache( std::vector const& argv) { // [START storage_control_disable_anywhere_cache] namespace storagecontrol = google::cloud::storagecontrol_v2; - [](storagecontrol::StorageControlClient client, std::string cache_name) { + [](storagecontrol::StorageControlClient client, + std::string const& cache_name) { auto anywhere_cache = client.DisableAnywhereCache(cache_name); if (!anywhere_cache) throw std::move(anywhere_cache).status(); std::cout << "Disabled anywhere cache: " << anywhere_cache->name() << "\n";