Cloud Storage上の大量のオブジェクトをバッチリクエストで削除してみた

# Batch APIを用いた実装イメージ
with client.batch():
    for blob in chunk:
        blob.delete()

from google.cloud import storage
from flask import jsonify, Request
import functions_framework
import time  # 時間計測のために追加

@functions_framework.http
def delete_objects_in_gcs(request: Request):
    try:
        # リクエストからパラメータを取得
        request_json = request.get_json()
        if not request_json:
            return jsonify({"error": "Invalid request, JSON body is required."}), 400

        bucket_name = request_json.get("bucket_name")  # バケット名をリクエストから取得
        if not bucket_name:
            return jsonify({"error": "Missing 'bucket_name' in request."}), 400

        method = request_json.get("method", "batch")  # デフォルトは "batch"
        chunk_size = 100  # 固定で100件ずつ削除

        # クライアントを初期化
        client = storage.Client()
        bucket = client.bucket(bucket_name)

        # バケット内のすべてのオブジェクトを取得
        blobs = list(bucket.list_blobs())

        if not blobs:
            return jsonify({"message": f"No objects found in bucket '{bucket_name}'."}), 200

        # オブジェクトをチャンクに分割
        chunks = [blobs[i:i + chunk_size] for i in range(0, len(blobs), chunk_size)]

        # 削除処理の開始時刻を記録
        start_time = time.time()

        # 削除方法に応じて処理を分岐
        if method == "batch":
            # バッチ削除
            for i, chunk in enumerate(chunks):
                with client.batch():
                    for blob in chunk:
                        blob.delete()
                print(f"Batch {i + 1}/{len(chunks)}: Deleted {len(chunk)} objects.")
        elif method == "sequential":
            # 順次削除
            for i, blob in enumerate(blobs):
                blob.delete()
                if (i + 1) % chunk_size == 0 or (i + 1) == len(blobs):
                    print(f"Sequential: Deleted {i + 1}/{len(blobs)} objects.")
        else:
            return jsonify({"error": f"Invalid method '{method}'. Use 'batch' or 'sequential'."}), 400

        # 削除処理の終了時刻を記録
        end_time = time.time()

        # 削除処理にかかった時間を計算
        elapsed_time = end_time - start_time

        return jsonify({
            "message": f"Successfully deleted {len(blobs)} objects from bucket '{bucket_name}' using method '{method}'.",
            "elapsed_time_seconds": elapsed_time
        }), 200

    except Exception as e:
        print(f"Error: {e}")
        return jsonify({"error": str(e)}), 500

curl -X POST Cloud Run関数のURL \
-H "Content-Type: application/json" \
-d '{
  "bucket_name": "バケット名",
  "method": "batch"
}'

curl -X POST Cloud Run関数のURL \
-H "Content-Type: application/json" \
-d '{
  "bucket_name": "バケット名",
  "method": "sequential"
}'

chunks = [blobs[i:i + chunk_size] for i in range(0, len(blobs), chunk_size)]

if method == "batch":
    for i, chunk in enumerate(chunks):
        with client.batch():
            for blob in chunk:
                blob.delete()
        print(f"Batch {i + 1}/{len(chunks)}: Deleted {len(chunk)} objects.")

elif method == "sequential":
    for i, blob in enumerate(blobs):
        blob.delete()
        if (i + 1) % chunk_size == 0 or (i + 1) == len(blobs):
            print(f"Sequential: Deleted {i + 1}/{len(blobs)} objects.")