diff --git a/official/legacy/bert/bert_cloud_tpu.md b/official/legacy/bert/bert_cloud_tpu.md index baf6f9bdc0c..90e5bd9f02a 100644 --- a/official/legacy/bert/bert_cloud_tpu.md +++ b/official/legacy/bert/bert_cloud_tpu.md @@ -14,7 +14,7 @@ gcloud config set project ${PROJECT_ID} ``` 4. Create a Cloud Storage bucket using the following command: ``` -gsutil mb -p ${PROJECT_ID} -c standard -l europe-west4 -b on gs://your-bucket-name +gcloud storage buckets create gs://your-bucket-name --project ${PROJECT_ID} --default-storage-class standard --location europe-west4 --uniform-bucket-level-access ``` This Cloud Storage bucket stores the data you use to train your model and the training results. 5. Launch a Compute Engine VM and Cloud TPU using the ctpu up command. @@ -98,9 +98,9 @@ $ ctpu delete --zone=your-zone ``` $ ctpu status --zone=your-zone ``` -4. Run gsutil as shown, replacing your-bucket with the name of the Cloud Storage bucket you created for this tutorial: +4. Run gcloud storage as shown, replacing your-bucket with the name of the Cloud Storage bucket you created for this tutorial: ``` -$ gsutil rm -r gs://your-bucket +$ gcloud storage rm --recursive gs://your-bucket ``` diff --git a/official/projects/edgetpu/vision/serving/inference_visualization_tool.ipynb b/official/projects/edgetpu/vision/serving/inference_visualization_tool.ipynb index e163df3052d..f0795773c03 100644 --- a/official/projects/edgetpu/vision/serving/inference_visualization_tool.ipynb +++ b/official/projects/edgetpu/vision/serving/inference_visualization_tool.ipynb @@ -57,7 +57,7 @@ " sandbox_path = web_path.split('/')[-1]\n", " !rm -f {sandbox_path}\n", " if web_path[:2] == \"gs\":\n", - " !gsutil cp {web_path} {sandbox_path}\n", + " !gcloud storage cp {web_path} {sandbox_path}\n", " else:\n", " !wget -v {web_path} --no-check-certificate\n", " return sandbox_path\n" @@ -122,7 +122,7 @@ }, "source": [ "MODEL_HOME='gs://tf_model_garden/models/edgetpu/checkpoint_and_tflite/vision/segmentation-edgetpu/tflite/default_argmax'\n", - "!gsutil ls {MODEL_HOME}" + "!gcloud storage ls {MODEL_HOME}" ], "execution_count": null, "outputs": [] diff --git a/official/projects/longformer/README.md b/official/projects/longformer/README.md index 8db9a3ac8fb..641d22b3b72 100644 --- a/official/projects/longformer/README.md +++ b/official/projects/longformer/README.md @@ -21,7 +21,7 @@ pytorch longformer tokenized data to tf_records. Option 1. Use our saved checkpoint of `allenai/longformer-base-4096` stored in cloud storage ```bash -gsutil cp -r gs://model-garden-ucsd-zihan/longformer-4096 . +gcloud storage cp --recursive gs://model-garden-ucsd-zihan/longformer-4096 . ``` Option 2. Create it directly diff --git a/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/big_query_ops.py b/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/big_query_ops.py index 4b8aeac3a8f..20b926b68a6 100644 --- a/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/big_query_ops.py +++ b/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/big_query_ops.py @@ -131,7 +131,7 @@ def upload_image_results_to_storage_bucket( try: commands = [ f"rm -r {os.path.basename(input_directory)}", - f"gsutil -m cp -r {prediction_folder} {output_directory}", + f"gcloud storage cp --recursive {prediction_folder} {output_directory}", f"rm -r {prediction_folder}", ] subprocess.run(" && ".join(commands), shell=True, check=True) diff --git a/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/big_query_ops_test.py b/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/big_query_ops_test.py index c0fc0f35170..0d2ee7084b7 100644 --- a/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/big_query_ops_test.py +++ b/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/big_query_ops_test.py @@ -198,7 +198,7 @@ def test_upload_image_results_to_storage_bucket_success(self): self.mock_subprocess_run.assert_called_once() args, _ = self.mock_subprocess_run.call_args - self.assertIn(f"gsutil -m cp -r {pred_dir} {output_dir}", args[0]) + self.assertIn(f"gcloud storage cp --recursive {pred_dir} {output_dir}", args[0]) def test_upload_image_results_to_storage_bucket_failure(self): input_dir = "/tmp/input" diff --git a/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/utils.py b/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/utils.py index 9b9194d7c88..29546c1a5c2 100644 --- a/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/utils.py +++ b/official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/utils.py @@ -94,7 +94,7 @@ def setup_logger_and_directories(input_dir): """ input_directory = (input_dir).rstrip('/\\') - command = f'gsutil -m cp -r {input_directory} .' + command = f'gcloud storage cp --recursive {input_directory} .' subprocess.run(command, shell=True, check=True) prediction_folder = os.path.basename(input_directory) + '_prediction' os.makedirs(prediction_folder, exist_ok=True) diff --git a/official/projects/waste_identification_ml/llm_applications/milk_pouch_detection/README.md b/official/projects/waste_identification_ml/llm_applications/milk_pouch_detection/README.md index a5bfb4be582..e43fe129a94 100644 --- a/official/projects/waste_identification_ml/llm_applications/milk_pouch_detection/README.md +++ b/official/projects/waste_identification_ml/llm_applications/milk_pouch_detection/README.md @@ -100,7 +100,7 @@ images to the source GCS bucket created by the script. The script output will provide the name of the bucket. ```bash -gsutil cp your-local-image.jpg gs:/// +gcloud storage cp your-local-image.jpg gs:/// ``` --- diff --git a/official/projects/waste_identification_ml/llm_applications/milk_pouch_detection/src/run_pipeline.sh b/official/projects/waste_identification_ml/llm_applications/milk_pouch_detection/src/run_pipeline.sh index ce958c05157..8fc65260ae4 100755 --- a/official/projects/waste_identification_ml/llm_applications/milk_pouch_detection/src/run_pipeline.sh +++ b/official/projects/waste_identification_ml/llm_applications/milk_pouch_detection/src/run_pipeline.sh @@ -29,14 +29,14 @@ cd milk_pouch_project # NOTE: Adjust the grep pattern if other image types are expected. echo "=== DEBUGGING START ===" echo "DEBUG: gcs_path variable is: '${gcs_path}'" -echo "DEBUG: Running 'gsutil ls \"${gcs_path}\"' to check accessibility:" -gsutil ls "${gcs_path}" || echo "❌ gsutil ls failed" -echo "DEBUG: Running 'gsutil ls -r \"${gcs_path}\" | head -n 10' to check content:" -gsutil ls -r "${gcs_path}" | head -n 10 || echo "❌ gsutil recursive ls failed" +echo "DEBUG: Running 'gcloud storage ls "${gcs_path}"' to check accessibility:" +gcloud storage ls "${gcs_path}" || echo "❌ gsutil ls failed" +echo "DEBUG: Running 'gcloud storage ls --recursive "${gcs_path}" | head -n 10' to check content:" +gcloud storage ls --recursive "${gcs_path}" | head -n 10 || echo "❌ gsutil recursive ls failed" echo "=== DEBUGGING END ===" echo "🖨️ Listing image files from GCS bucket: $gcs_path" -mapfile -t all_gcs_files < <(gsutil ls -r "${gcs_path}" | grep -iE '\.(png|jpg|jpeg)$' | grep -v "/predictions/" | grep -v "/processed/") +mapfile -t all_gcs_files < <(gcloud storage ls --recursive "${gcs_path}" | grep -iE '\.(png|jpg|jpeg)$' | grep -v "/predictions/" | grep -v "/processed/") num_files=${#all_gcs_files[@]} if (( num_files == 0 )); then @@ -77,7 +77,7 @@ for (( i=0; i 0 )); then - gsutil -m mv "${current_move_batch[@]}" "$current_move_dir" + gcloud storage mv "${current_move_batch[@]}" "$current_move_dir" current_move_batch=() fi current_move_dir="$dest_dir" @@ -138,7 +138,7 @@ for (( i=0; i 0 )); then - gsutil -m mv "${current_move_batch[@]}" "$current_move_dir" + gcloud storage mv "${current_move_batch[@]}" "$current_move_dir" fi unset current_move_batch diff --git a/research/object_detection/dockerfiles/tf2_ai_platform/Dockerfile b/research/object_detection/dockerfiles/tf2_ai_platform/Dockerfile index 0b43e0620f7..7427c37f54d 100644 --- a/research/object_detection/dockerfiles/tf2_ai_platform/Dockerfile +++ b/research/object_detection/dockerfiles/tf2_ai_platform/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update && apt-get install -y \ python3-opencv \ wget -# Installs google cloud sdk, this is mostly for using gsutil to export model. +# Installs google cloud sdk, this is mostly for using gcloud storage to export model. RUN wget -nv \ https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz && \ mkdir /root/tools && \ @@ -29,7 +29,7 @@ RUN wget -nv \ # Path configuration ENV PATH $PATH:/root/tools/google-cloud-sdk/bin -# Make sure gsutil will use the default service account +# Make sure gcloud storage will use the default service account RUN echo '[GoogleCompute]\nservice_account = default' > /etc/boto.cfg WORKDIR /home/tensorflow diff --git a/research/object_detection/g3doc/oid_inference_and_evaluation.md b/research/object_detection/g3doc/oid_inference_and_evaluation.md index d54ad23940b..f6d38b52953 100644 --- a/research/object_detection/g3doc/oid_inference_and_evaluation.md +++ b/research/object_detection/g3doc/oid_inference_and_evaluation.md @@ -58,7 +58,7 @@ access to the cloud bucket with the images. Then run: # From tensorflow/models/research/oid SPLIT=validation # Set SPLIT to "test" to download the images in the test set mkdir raw_images_${SPLIT} -gsutil -m rsync -r gs://open-images-dataset/$SPLIT raw_images_${SPLIT} +gcloud storage rsync --recursive gs://open-images-dataset/$SPLIT raw_images_${SPLIT} ``` Another option for downloading the images is to follow the URLs contained in the