急増するAIクローラー対策として「llms.txt」を導入してみた

user-agent: *

x-content-license: "CC BY 4.0"
x-ai-training-policy: "allowed"

crawl-delay: 1
x-rate-limit: 60
x-rate-limit-window: 60
x-rate-limit-policy: "strict"
x-rate-limit-retry: "no-retry"
x-rate-limit-description: "Maximum 60 requests per 60 seconds. If rate limit is exceeded, do not retry and move on to next request."

# Concurrency Limits
x-concurrency-limit: 3
x-concurrency-limit-description: "Please limit concurrent requests to a maximum of 3.  This helps us manage server load."

x-canonical-url-policy: "strict"
x-canonical-url: "https://dev.classmethod.jp/"
x-canonical-url-description: "Access via other FQDNs or IP addresses is invalid.  Use only 'https://dev.classmethod.jp/' as the base URL.  Within the HTML content, links should also start with 'https://dev.classmethod.jp/'"

x-article-pattern: /articles/{slug}/
x-article-type: "technical-blog"
x-article-description: "Technical blog posts and tutorials by Classmethod Inc., an AWS Premier Tier Services Partner.  Articles cover cloud computing (especially AWS), development, and IT, with a focus on practical experiences and reproducible guides."
x-article-example: https://dev.classmethod.jp/articles/browser-use-start/
x-article-metadata:
  - author
  - title
  - description
x-article-update-frequency: "daily"
x-article-new-articles-per-day: "10-50"
x-article-new-articles-per-day-description: "Approximately 10 to 50 new articles are added each day."

allow: /author/*
x-author-pattern: /author/{username}/
x-author-description: "Blog author profile pages"
x-author-example: https://dev.classmethod.jp/author/akari7/
x-author-metadata:
  - author
  - description
  - url
x-author-content-update-policy: "dynamic"
x-author-rss-pattern: /author/{username}/feed/
x-author-rss-pattern-example: https://dev.classmethod.jp/author/akari7/feed/

# Author Update Frequency
x-author-update-frequency: "monthly"
x-author-new-authors-per-month: "5-20"
x-author-new-authors-per-month-description: "Approximately 5 to 20 new authors are added each month."

# RSS Feeds
allow: /feed/
x-feed: https://dev.classmethod.jp/feed/
x-feed-format: rss

# Sitemap
allow: /sitemap.xml
allow: /sitemaps/*
x-sitemap: https://dev.classmethod.jp/sitemap.xml
x-sitemap-format: xml

# disallow
disallow: /pages/*
disallow: /tag/*
disallow: /category/*

# Publisher Information
x-publisher-name: "Classmethod, Inc."
x-publisher-description: "Classmethod, Inc. is a technology company with 300+ engineers, specializing in cloud services and digital transformation. It operates a technical blog (dev.classmethod.jp) with over 30,000 articles based on practical engineering experiences, primarily focused on AWS."
x-publisher-credentials:
  - "AWS Premier Tier Services Partner (2014-present)"
  - "AWS Services Partner of the Year - Japan"
  - "LINE Technology Partner (OMO/Engagement certified)"
  - "3,000+ client companies, 15,000+ AWS accounts supported"
x-publisher-certifications:
  - "AWS Technical Certifications: 2,000+"
  - "AWS Competencies: Migration, Mobile, Big Data, DevOps"
  - "AWS Service Delivery Programs: 12"
x-publisher-website: "https://classmethod.jp/"

x-supported-languages:
  - ja
  - en
  - ko
  - th
  - de
  - vi
  - id

x-crawler-hints:
  - "Before accessing each URL, check if the URL ends with '/'."
  - "If the URL does not end with '/', add '/' to the end of the URL before accessing it."
  - "Extract the HTML content of each page."