pyarrowでCloud Storage 上の Parquet ファイルのスキーマ情報を取得した場合、ファイル全量ではなくメタデータのみ取得しているか確認してみた

import pyarrow.parquet as pq
import pyarrow.fs as fs
import gcsfs
import logging
import time
import tracemalloc

bucket_name = "バケット名"
object_path = "sample_1gb.parquet"

gcs = fs.GcsFileSystem()


# 実行時間計測
start_time = time.time()
tracemalloc.start()  # メモリ使用量の計測始

file_info = gcs.get_file_info(f"{bucket_name}/{object_path}")
file_size = file_info.size
print(f"Total file size: {file_size / (1024 * 1024):.2f} MB")

# gcsfs のデバッグログを有効化
logging.basicConfig(level=logging.DEBUG)

# gcsfs を使用してCloud Storageへのリクエストを監視
fs_gcs = gcsfs.GCSFileSystem()

print("Fetching Parquet metadata...")
parquet_file = pq.ParquetFile(f"{bucket_name}/{object_path}", filesystem=gcs)

print("カラム名: 物理型, 論理型")
for column in parquet_file.metadata.schema:
    print(f"{column.name}: {column.physical_type}, ({column.logical_type})")

# `gcsfs` のリクエストログを取得
with fs_gcs.open(f"gs://{bucket_name}/{object_path}", "rb") as f:
    f.read(100)  # 100バイトだけ読み込む（ログを確認するため）

end_time = time.time()
current, peak = tracemalloc.get_traced_memory()
tracemalloc.stop()

print("\nExecution Metrics:")
print(f"Execution time: {end_time - start_time:.4f} seconds")
print(f"Memory usage: {current / (1024 * 1024):.2f} MB (Peak: {peak / (1024 * 1024):.2f} MB)")

% python check_parquet.py
Total file size: 1397.65 MB
DEBUG:asyncio:Using selector: KqueueSelector
DEBUG:google.auth._default:Checking None for explicit credentials as part of auth process...
DEBUG:google.auth._default:Checking Cloud SDK credentials as part of auth process...
DEBUG:gcsfs.credentials:Connected with method google_default
Fetching Parquet metadata...
カラム名: 物理型, 論理型
id: INT64, (None)
name: BYTE_ARRAY, (String)
age: INT64, (None)
salary: DOUBLE, (None)
description: BYTE_ARRAY, (String)
index_level_0: INT64, (None)
DEBUG:gcsfs:GET: b/{}/o/{}, ('バケット名', 'sample_1gb.parquet'), None
DEBUG:gcsfs.credentials:GCS refresh
DEBUG:google.auth.transport.requests:Making request: POST https://oauth2.googleapis.com/token
DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): oauth2.googleapis.com:443
DEBUG:urllib3.connectionpool:https://oauth2.googleapis.com:443 "POST /token HTTP/1.1" 200 None
DEBUG:gcsfs:GET: https://storage.googleapis.com/download/storage/v1/b/バケット名/o/sample_1gb.parquet?alt=media, (), {'Range': 'bytes=0-5242979'}
DEBUG:fsspec:<File-like object GCSFileSystem, バケット名/sample_1gb.parquet> read: 0 - 100  , readahead: 0 hits, 1 misses, 5242980 total requested bytes

Execution Metrics:
Execution time: 1.9332 seconds
Memory usage: 0.78 MB (Peak: 10.79 MB)

Total file size: 1397.65 MB

Fetching Parquet metadata...
カラム名: 物理型, 論理型
id: INT64, (None)
name: BYTE_ARRAY, (String)
age: INT64, (None)
salary: DOUBLE, (None)
description: BYTE_ARRAY, (String)
__index_level_0__: INT64, (None)

DEBUG:gcsfs:GET: b/{}/o/{}, ('バケット名', 'sample_1gb.parquet'), None
DEBUG:gcsfs.credentials:GCS refresh
DEBUG:google.auth.transport.requests:Making request: POST https://oauth2.googleapis.com/token
DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): oauth2.googleapis.com:443
DEBUG:urllib3.connectionpool:https://oauth2.googleapis.com:443 "POST /token HTTP/1.1" 200 None

DEBUG:gcsfs:GET: https://storage.googleapis.com/download/storage/v1/b/バケット名/o/sample_1gb.parquet?alt=media, (), {'Range': 'bytes=0-5242979'}

Execution time: 1.9332 seconds

Memory usage: 0.78 MB (Peak: 10.79 MB)

# "salary" 列のみを取得
salary_column = parquet_file.read(columns=["salary"]).to_pandas()
# 列の合計を計算
total_salary = salary_column["salary"].sum()
print(f"Total Salary: {total_salary}")

Total file size: 1397.65 MB
DEBUG:asyncio:Using selector: KqueueSelector
DEBUG:google.auth._default:Checking None for explicit credentials as part of auth process...
DEBUG:google.auth._default:Checking Cloud SDK credentials as part of auth process...
DEBUG:gcsfs.credentials:Connected with method google_default
Fetching 'salary' column data and calculating sum...
Total Salary: 7199860009243.785

Checking GCS request details for 'salary' column...
DEBUG:gcsfs:GET: b/{}/o/{}, ('バケット名', 'sample_1gb.parquet'), None
DEBUG:gcsfs.credentials:GCS refresh
DEBUG:google.auth.transport.requests:Making request: POST https://oauth2.googleapis.com/token
DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): oauth2.googleapis.com:443
DEBUG:urllib3.connectionpool:https://oauth2.googleapis.com:443 "POST /token HTTP/1.1" 200 None
DEBUG:gcsfs:GET: https://storage.googleapis.com/download/storage/v1/b/バケット名/o/sample_1gb.parquet?alt=media, (), {'Range': 'bytes=0-5242979'}
DEBUG:fsspec:<File-like object GCSFileSystem, バケット名/sample_1gb.parquet> read: 0 - 100  , readahead: 0 hits, 1 misses, 5242980 total requested bytes

Execution Metrics:
Execution time: 22.6811 seconds
Memory usage: 21.24 MB (Peak: 96.88 MB)