{
 "@context": [
  "http://www.w3.org/ns/csvw",
  {
   "@language": "en"
  }
 ],
 "url": "ai-crawler-study-2026-10.csv",
 "dc:title": "AI crawler rules study, October 2026: aggregate counts",
 "dc:description": "Counts of websites in the CrUX top lists for Germany, Austria, Switzerland (up to rank 10,000, 5,000 and 5,000) and the global top 10,000, by the rules their robots.txt, llms.txt and ai.txt files set for 15 AI crawlers. Aggregates only. No website is named. Measure definitions: see the method section of https://www.paceghost.io/en/resources/ai-crawler-study/",
 "dc:creator": "paceghost.io",
 "dc:publisher": {
  "@id": "https://www.paceghost.io/"
 },
 "dc:issued": {
  "@value": "2026-10-13",
  "@type": "xsd:date"
 },
 "dc:license": {
  "@id": "https://creativecommons.org/licenses/by/4.0/"
 },
 "dc:rights": "CC BY 4.0. Provided as is, without warranties.",
 "dc:bibliographicCitation": "paceghost.io, AI crawler rules study, October 2026, https://www.paceghost.io/en/resources/ai-crawler-study/",
 "dcat:landingPage": {
  "@id": "https://www.paceghost.io/en/resources/ai-crawler-study/"
 },
 "prov:wasDerivedFrom": [
  {
   "@id": "https://developer.chrome.com/docs/crux/bigquery",
   "dc:title": "Chrome UX Report (CrUX), month 2026-08, BigQuery tables chrome-ux-report.experimental.country and chrome-ux-report.experimental.global, read on 2026-10-14",
   "dc:creator": "Google",
   "dc:license": {
    "@id": "https://creativecommons.org/licenses/by/4.0/"
   }
  }
 ],
 "rdfs:comment": "Changes to the source data: we combined the four lists (DE, AT, CH, global) and removed duplicate hosts. We requested robots.txt from each host, then llms.txt and ai.txt where robots.txt allowed it, and counted the results. This file holds the counts only, not the website lists.",
 "dialect": {
  "header": true,
  "delimiter": ","
 },
 "tableSchema": {
  "columns": [
   {
    "name": "run_date",
    "titles": "run_date",
    "datatype": "date",
    "dc:description": "Day the files were read (UTC)."
   },
   {
    "name": "group",
    "titles": "group",
    "datatype": "string",
    "dc:description": "Website group: DE, AT, CH (one country list), DACH (any of the three), DACH_only (DACH and not in the global top 10,000), Global (global top 10,000), All (every host)."
   },
   {
    "name": "rank_bucket",
    "titles": "rank_bucket",
    "datatype": "string",
    "dc:description": "CrUX rank bucket: all, 1-1000, 1001-5000 or 5001-10000."
   },
   {
    "name": "measure",
    "titles": "measure",
    "datatype": "string",
    "dc:description": "What is counted. Definitions: method section of the study page."
   },
   {
    "name": "crawler",
    "titles": "crawler",
    "datatype": "string",
    "dc:description": "AI crawler user-agent token for crawler_* measures. Empty otherwise."
   },
   {
    "name": "count",
    "titles": "count",
    "datatype": "integer",
    "dc:description": "Number of websites (distinct hosts) that meet the measure."
   },
   {
    "name": "base",
    "titles": "base",
    "datatype": "integer",
    "dc:description": "Denominator: number of websites the share is computed on. Empty when there is no share."
   },
   {
    "name": "share_percent",
    "titles": "share_percent",
    "datatype": "decimal",
    "dc:description": "100 * count / base, rounded to 2 decimals. Empty when there is no base."
   }
  ]
 }
}