File: rfc9309.html

package info (click to toggle)
doc-rfc 20230121-1
links: PTS, VCS
area: non-free
in suites: bookworm, forky, sid, trixie
size: 1,609,944 kB
file content (2149 lines) | stat: -rw-r--r-- 82,367 bytes
<!DOCTYPE html>
<html lang="en" class="RFC">
<head>
<meta charset="utf-8">
<meta content="Common,Latin" name="scripts">
<meta content="initial-scale=1.0" name="viewport">
<title>RFC 9309: Robots Exclusion Protocol</title>
<meta content="Martijn Koster" name="author">
<meta content="Gary Illyes" name="author">
<meta content="Henner Zeller" name="author">
<meta content="Lizzi Sassman" name="author">
<meta content='
        This document specifies and extends the "Robots Exclusion Protocol"
          method originally defined by Martijn Koster in 1994 for service owners
          to control how content served by their services may be accessed, if at
          all, by automatic clients known as crawlers. Specifically, it adds
          definition language for the protocol, instructions for handling
          errors, and instructions for caching.  
    ' name="description">
<meta content="xml2rfc 3.14.2" name="generator">
<meta content="robot" name="keyword">
<meta content="crawler" name="keyword">
<meta content="robots.txt" name="keyword">
<meta content="9309" name="rfc.number">
<!-- Generator version information:
  xml2rfc 3.14.2
    Python 3.9.13
    appdirs 1.4.4
    ConfigArgParse 1.5.3
    google-i18n-address 2.5.1
    html5lib 1.1
    intervaltree 3.1.0
    Jinja2 3.1.2
    kitchen 1.2.6
    lxml 4.9.0
    MarkupSafe 2.1.1
    pycountry 22.3.5
    PyYAML 6.0
    requests 2.28.0
    setuptools 44.1.1
    six 1.16.0
    weasyprint 56.1
-->
<link href="rfc9309.xml" rel="alternate" type="application/rfc+xml">
<link href="#copyright" rel="license">
<style type="text/css">/*

  NOTE: Changes at the bottom of this file overrides some earlier settings.

  Once the style has stabilized and has been adopted as an official RFC style,
  this can be consolidated so that style settings occur only in one place, but
  for now the contents of this file consists first of the initial CSS work as
  provided to the RFC Formatter (xml2rfc) work, followed by itemized and
  commented changes found necssary during the development of the v3
  formatters.

*/

/* fonts */
@import url('https://fonts.googleapis.com/css?family=Noto+Sans'); /* Sans-serif */
@import url('https://fonts.googleapis.com/css?family=Noto+Serif'); /* Serif (print) */
@import url('https://fonts.googleapis.com/css?family=Roboto+Mono'); /* Monospace */

@viewport {
  zoom: 1.0;
  width: extend-to-zoom;
}
@-ms-viewport {
  width: extend-to-zoom;
  zoom: 1.0;
}
/* general and mobile first */
html {
}
body {
  max-width: 90%;
  margin: 1.5em auto;
  color: #222;
  background-color: #fff;
  font-size: 14px;
  font-family: 'Noto Sans', Arial, Helvetica, sans-serif;
  line-height: 1.6;
  scroll-behavior: smooth;
}
.ears {
  display: none;
}

/* headings */
#title, h1, h2, h3, h4, h5, h6 {
  margin: 1em 0 0.5em;
  font-weight: bold;
  line-height: 1.3;
}
#title {
  clear: both;
  border-bottom: 1px solid #ddd;
  margin: 0 0 0.5em 0;
  padding: 1em 0 0.5em;
}
.author {
  padding-bottom: 4px;
}
h1 {
  font-size: 26px;
  margin: 1em 0;
}
h2 {
  font-size: 22px;
  margin-top: -20px;  /* provide offset for in-page anchors */
  padding-top: 33px;
}
h3 {
  font-size: 18px;
  margin-top: -36px;  /* provide offset for in-page anchors */
  padding-top: 42px;
}
h4 {
  font-size: 16px;
  margin-top: -36px;  /* provide offset for in-page anchors */
  padding-top: 42px;
}
h5, h6 {
  font-size: 14px;
}
#n-copyright-notice {
  border-bottom: 1px solid #ddd;
  padding-bottom: 1em;
  margin-bottom: 1em;
}
/* general structure */
p {
  padding: 0;
  margin: 0 0 1em 0;
  text-align: left;
}
div, span {
  position: relative;
}
div {
  margin: 0;
}
.alignRight.art-text {
  background-color: #f9f9f9;
  border: 1px solid #eee;
  border-radius: 3px;
  padding: 1em 1em 0;
  margin-bottom: 1.5em;
}
.alignRight.art-text pre {
  padding: 0;
}
.alignRight {
  margin: 1em 0;
}
.alignRight > *:first-child {
  border: none;
  margin: 0;
  float: right;
  clear: both;
}
.alignRight > *:nth-child(2) {
  clear: both;
  display: block;
  border: none;
}
svg {
  display: block;
}
.alignCenter.art-text {
  background-color: #f9f9f9;
  border: 1px solid #eee;
  border-radius: 3px;
  padding: 1em 1em 0;
  margin-bottom: 1.5em;
}
.alignCenter.art-text pre {
  padding: 0;
}
.alignCenter {
  margin: 1em 0;
}
.alignCenter > *:first-child {
  display: table;
  border: none;
  margin: 0 auto;
}

/* lists */
ol, ul {
  padding: 0;
  margin: 0 0 1em 2em;
}
ol ol, ul ul, ol ul, ul ol {
  margin-left: 1em;
}
li {
  margin: 0 0 0.25em 0;
}
.ulCompact li {
  margin: 0;
}
ul.empty, .ulEmpty {
  list-style-type: none;
}
ul.empty li, .ulEmpty li {
  margin-top: 0.5em;
}
ul.ulBare, li.ulBare {
  margin-left: 0em !important;
}
ul.compact, .ulCompact,
ol.compact, .olCompact {
  line-height: 100%;
  margin: 0 0 0 2em;
}

/* definition lists */
dl {
}
dl > dt {
  float: left;
  margin-right: 1em;
}
/* 
dl.nohang > dt {
  float: none;
}
*/
dl > dd {
  margin-bottom: .8em;
  min-height: 1.3em;
}
dl.compact > dd, .dlCompact > dd {
  margin-bottom: 0em;
}
dl > dd > dl {
  margin-top: 0.5em;
  margin-bottom: 0em;
}

/* links */
a {
  text-decoration: none;
}
a[href] {
  color: #22e; /* Arlen: WCAG 2019 */
}
a[href]:hover {
  background-color: #f2f2f2;
}
figcaption a[href],
a[href].selfRef {
  color: #222;
}
/* XXX probably not this:
a.selfRef:hover {
  background-color: transparent;
  cursor: default;
} */

/* Figures */
tt, code, pre, code {
  background-color: #f9f9f9;
  font-family: 'Roboto Mono', monospace;
}
pre {
  border: 1px solid #eee;
  margin: 0;
  padding: 1em;
}
img {
  max-width: 100%;
}
figure {
  margin: 0;
}
figure blockquote {
  margin: 0.8em 0.4em 0.4em;
}
figcaption {
  font-style: italic;
  margin: 0 0 1em 0;
}
@media screen {
  pre {
    overflow-x: auto;
    max-width: 100%;
    max-width: calc(100% - 22px);
  }
}

/* aside, blockquote */
aside, blockquote {
  margin-left: 0;
  padding: 1.2em 2em;
}
blockquote {
  background-color: #f9f9f9;
  color: #111; /* Arlen: WCAG 2019 */
  border: 1px solid #ddd;
  border-radius: 3px;
  margin: 1em 0;
}
cite {
  display: block;
  text-align: right;
  font-style: italic;
}

/* tables */
table {
  width: 100%;
  margin: 0 0 1em;
  border-collapse: collapse;
  border: 1px solid #eee;
}
th, td {
  text-align: left;
  vertical-align: top;
  padding: 0.5em 0.75em;
}
th {
  text-align: left;
  background-color: #e9e9e9;
}
tr:nth-child(2n+1) > td {
  background-color: #f5f5f5;
}
table caption {
  font-style: italic;
  margin: 0;
  padding: 0;
  text-align: left;
}
table p {
  /* XXX to avoid bottom margin on table row signifiers. If paragraphs should
     be allowed within tables more generally, it would be far better to select on a class. */
  margin: 0;
}

/* pilcrow */
a.pilcrow {
  color: #666; /* Arlen: AHDJ 2019 */
  text-decoration: none;
  visibility: hidden;
  user-select: none;
  -ms-user-select: none;
  -o-user-select:none;
  -moz-user-select: none;
  -khtml-user-select: none;
  -webkit-user-select: none;
  -webkit-touch-callout: none;
}
@media screen {
  aside:hover > a.pilcrow,
  p:hover > a.pilcrow,
  blockquote:hover > a.pilcrow,
  div:hover > a.pilcrow,
  li:hover > a.pilcrow,
  pre:hover > a.pilcrow {
    visibility: visible;
  }
  a.pilcrow:hover {
    background-color: transparent;
  }
}

/* misc */
hr {
  border: 0;
  border-top: 1px solid #eee;
}
.bcp14 {
  font-variant: small-caps;
}

.role {
  font-variant: all-small-caps;
}

/* info block */
#identifiers {
  margin: 0;
  font-size: 0.9em;
}
#identifiers dt {
  width: 3em;
  clear: left;
}
#identifiers dd {
  float: left;
  margin-bottom: 0;
}
/* Fix PDF info block run off issue */
@media print {
  #identifiers dd {
    float: none;
  }
}
#identifiers .authors .author {
  display: inline-block;
  margin-right: 1.5em;
}
#identifiers .authors .org {
  font-style: italic;
}

/* The prepared/rendered info at the very bottom of the page */
.docInfo {
  color: #666; /* Arlen: WCAG 2019 */
  font-size: 0.9em;
  font-style: italic;
  margin-top: 2em;
}
.docInfo .prepared {
  float: left;
}
.docInfo .prepared {
  float: right;
}

/* table of contents */
#toc  {
  padding: 0.75em 0 2em 0;
  margin-bottom: 1em;
}
nav.toc ul {
  margin: 0 0.5em 0 0;
  padding: 0;
  list-style: none;
}
nav.toc li {
  line-height: 1.3em;
  margin: 0.75em 0;
  padding-left: 1.2em;
  text-indent: -1.2em;
}
/* references */
.references dt {
  text-align: right;
  font-weight: bold;
  min-width: 7em;
}
.references dd {
  margin-left: 8em;
  overflow: auto;
}

.refInstance {
  margin-bottom: 1.25em;
}

.references .ascii {
  margin-bottom: 0.25em;
}

/* index */
.index ul {
  margin: 0 0 0 1em;
  padding: 0;
  list-style: none;
}
.index ul ul {
  margin: 0;
}
.index li {
  margin: 0;
  text-indent: -2em;
  padding-left: 2em;
  padding-bottom: 5px;
}
.indexIndex {
  margin: 0.5em 0 1em;
}
.index a {
  font-weight: 700;
}
/* make the index two-column on all but the smallest screens */
@media (min-width: 600px) {
  .index ul {
    -moz-column-count: 2;
    -moz-column-gap: 20px;
  }
  .index ul ul {
    -moz-column-count: 1;
    -moz-column-gap: 0;
  }
}

/* authors */
address.vcard {
  font-style: normal;
  margin: 1em 0;
}

address.vcard .nameRole {
  font-weight: 700;
  margin-left: 0;
}
address.vcard .label {
  font-family: "Noto Sans",Arial,Helvetica,sans-serif;
  margin: 0.5em 0;
}
address.vcard .type {
  display: none;
}
.alternative-contact {
  margin: 1.5em 0 1em;
}
hr.addr {
  border-top: 1px dashed;
  margin: 0;
  color: #ddd;
  max-width: calc(100% - 16px);
}

/* temporary notes */
.rfcEditorRemove::before {
  position: absolute;
  top: 0.2em;
  right: 0.2em;
  padding: 0.2em;
  content: "The RFC Editor will remove this note";
  color: #9e2a00; /* Arlen: WCAG 2019 */
  background-color: #ffd; /* Arlen: WCAG 2019 */
}
.rfcEditorRemove {
  position: relative;
  padding-top: 1.8em;
  background-color: #ffd; /* Arlen: WCAG 2019 */
  border-radius: 3px;
}
.cref {
  background-color: #ffd; /* Arlen: WCAG 2019 */
  padding: 2px 4px;
}
.crefSource {
  font-style: italic;
}
/* alternative layout for smaller screens */
@media screen and (max-width: 1023px) {
  body {
    padding-top: 2em;
  }
  #title {
    padding: 1em 0;
  }
  h1 {
    font-size: 24px;
  }
  h2 {
    font-size: 20px;
    margin-top: -18px;  /* provide offset for in-page anchors */
    padding-top: 38px;
  }
  #identifiers dd {
    max-width: 60%;
  }
  #toc {
    position: fixed;
    z-index: 2;
    top: 0;
    right: 0;
    padding: 0;
    margin: 0;
    background-color: inherit;
    border-bottom: 1px solid #ccc;
  }
  #toc h2 {
    margin: -1px 0 0 0;
    padding: 4px 0 4px 6px;
    padding-right: 1em;
    min-width: 190px;
    font-size: 1.1em;
    text-align: right;
    background-color: #444;
    color: white;
    cursor: pointer;
  }
  #toc h2::before { /* css hamburger */
    float: right;
    position: relative;
    width: 1em;
    height: 1px;
    left: -164px;
    margin: 6px 0 0 0;
    background: white none repeat scroll 0 0;
    box-shadow: 0 4px 0 0 white, 0 8px 0 0 white;
    content: "";
  }
  #toc nav {
    display: none;
    padding: 0.5em 1em 1em;
    overflow: auto;
    height: calc(100vh - 48px);
    border-left: 1px solid #ddd;
  }
}

/* alternative layout for wide screens */
@media screen and (min-width: 1024px) {
  body {
    max-width: 724px;
    margin: 42px auto;
    padding-left: 1.5em;
    padding-right: 29em;
  }
  #toc {
    position: fixed;
    top: 42px;
    right: 42px;
    width: 25%;
    margin: 0;
    padding: 0 1em;
    z-index: 1;
  }
  #toc h2 {
    border-top: none;
    border-bottom: 1px solid #ddd;
    font-size: 1em;
    font-weight: normal;
    margin: 0;
    padding: 0.25em 1em 1em 0;
  }
  #toc nav {
    display: block;
    height: calc(90vh - 84px);
    bottom: 0;
    padding: 0.5em 0 0;
    overflow: auto;
  }
  img { /* future proofing */
    max-width: 100%;
    height: auto;
  }
}

/* pagination */
@media print {
  body {

    width: 100%;
  }
  p {
    orphans: 3;
    widows: 3;
  }
  #n-copyright-notice {
    border-bottom: none;
  }
  #toc, #n-introduction {
    page-break-before: always;
  }
  #toc {
    border-top: none;
    padding-top: 0;
  }
  figure, pre {
    page-break-inside: avoid;
  }
  figure {
    overflow: scroll;
  }
  pre.breakable {
    break-inside: auto;
  }
  h1, h2, h3, h4, h5, h6 {
    page-break-after: avoid;
  }
  h2+*, h3+*, h4+*, h5+*, h6+* {
    page-break-before: avoid;
  }
  pre {
    white-space: pre-wrap;
    word-wrap: break-word;
    font-size: 10pt;
  }
  table {
    border: 1px solid #ddd;
  }
  td {
    border-top: 1px solid #ddd;
  }
}

/* This is commented out here, as the string-set: doesn't
   pass W3C validation currently */
/*
.ears thead .left {
  string-set: ears-top-left content();
}

.ears thead .center {
  string-set: ears-top-center content();
}

.ears thead .right {
  string-set: ears-top-right content();
}

.ears tfoot .left {
  string-set: ears-bottom-left content();
}

.ears tfoot .center {
  string-set: ears-bottom-center content();
}

.ears tfoot .right {
  string-set: ears-bottom-right content();
}
*/

@page :first {
  padding-top: 0;
  @top-left {
    content: normal;
    border: none;
  }
  @top-center {
    content: normal;
    border: none;
  }
  @top-right {
    content: normal;
    border: none;
  }
}

@page {
  size: A4;
  margin-bottom: 45mm;
  padding-top: 20px;
  /* The follwing is commented out here, but set appropriately by in code, as
     the content depends on the document */
  /*
  @top-left {
    content: 'Internet-Draft';
    vertical-align: bottom;
    border-bottom: solid 1px #ccc;
  }
  @top-left {
    content: string(ears-top-left);
    vertical-align: bottom;
    border-bottom: solid 1px #ccc;
  }
  @top-center {
    content: string(ears-top-center);
    vertical-align: bottom;
    border-bottom: solid 1px #ccc;
  }
  @top-right {
    content: string(ears-top-right);
    vertical-align: bottom;
    border-bottom: solid 1px #ccc;
  }
  @bottom-left {
    content: string(ears-bottom-left);
    vertical-align: top;
    border-top: solid 1px #ccc;
  }
  @bottom-center {
    content: string(ears-bottom-center);
    vertical-align: top;
    border-top: solid 1px #ccc;
  }
  @bottom-right {
      content: '[Page ' counter(page) ']';
      vertical-align: top;
      border-top: solid 1px #ccc;
  }
  */

}

/* Changes introduced to fix issues found during implementation */
/* Make sure links are clickable even if overlapped by following H* */
a {
  z-index: 2;
}
/* Separate body from document info even without intervening H1 */
section {
  clear: both;
}


/* Top align author divs, to avoid names without organization dropping level with org names */
.author {
  vertical-align: top;
}

/* Leave room in document info to show Internet-Draft on one line */
#identifiers dt {
  width: 8em;
}

/* Don't waste quite as much whitespace between label and value in doc info */
#identifiers dd {
  margin-left: 1em;
}

/* Give floating toc a background color (needed when it's a div inside section */
#toc {
  background-color: white;
}

/* Make the collapsed ToC header render white on gray also when it's a link */
@media screen and (max-width: 1023px) {
  #toc h2 a,
  #toc h2 a:link,
  #toc h2 a:focus,
  #toc h2 a:hover,
  #toc a.toplink,
  #toc a.toplink:hover {
    color: white;
    background-color: #444;
    text-decoration: none;
  }
}

/* Give the bottom of the ToC some whitespace */
@media screen and (min-width: 1024px) {
  #toc {
    padding: 0 0 1em 1em;
  }
}

/* Style section numbers with more space between number and title */
.section-number {
  padding-right: 0.5em;
}

/* prevent monospace from becoming overly large */
tt, code, pre, code {
  font-size: 95%;
}

/* Fix the height/width aspect for ascii art*/
pre.sourcecode,
.art-text pre {
  line-height: 1.12;
}


/* Add styling for a link in the ToC that points to the top of the document */
a.toplink {
  float: right;
  margin-right: 0.5em;
}

/* Fix the dl styling to match the RFC 7992 attributes */
dl > dt,
dl.dlParallel > dt {
  float: left;
  margin-right: 1em;
}
dl.dlNewline > dt {
  float: none;
}

/* Provide styling for table cell text alignment */
table td.text-left,
table th.text-left {
  text-align: left;
}
table td.text-center,
table th.text-center {
  text-align: center;
}
table td.text-right,
table th.text-right {
  text-align: right;
}

/* Make the alternative author contact informatio look less like just another
   author, and group it closer with the primary author contact information */
.alternative-contact {
  margin: 0.5em 0 0.25em 0;
}
address .non-ascii {
  margin: 0 0 0 2em;
}

/* With it being possible to set tables with alignment
  left, center, and right, { width: 100%; } does not make sense */
table {
  width: auto;
}

/* Avoid reference text that sits in a block with very wide left margin,
   because of a long floating dt label.*/
.references dd {
  overflow: visible;
}

/* Control caption placement */
caption {
  caption-side: bottom;
}

/* Limit the width of the author address vcard, so names in right-to-left
   script don't end up on the other side of the page. */

address.vcard {
  max-width: 30em;
  margin-right: auto;
}

/* For address alignment dependent on LTR or RTL scripts */
address div.left {
  text-align: left;
}
address div.right {
  text-align: right;
}

/* Provide table alignment support.  We can't use the alignX classes above
   since they do unwanted things with caption and other styling. */
table.right {
 margin-left: auto;
 margin-right: 0;
}
table.center {
 margin-left: auto;
 margin-right: auto;
}
table.left {
 margin-left: 0;
 margin-right: auto;
}

/* Give the table caption label the same styling as the figcaption */
caption a[href] {
  color: #222;
}

@media print {
  .toplink {
    display: none;
  }

  /* avoid overwriting the top border line with the ToC header */
  #toc {
    padding-top: 1px;
  }

  /* Avoid page breaks inside dl and author address entries */
  .vcard {
    page-break-inside: avoid;
  }

}
/* Tweak the bcp14 keyword presentation */
.bcp14 {
  font-variant: small-caps;
  font-weight: bold;
  font-size: 0.9em;
}
/* Tweak the invisible space above H* in order not to overlay links in text above */
 h2 {
  margin-top: -18px;  /* provide offset for in-page anchors */
  padding-top: 31px;
 }
 h3 {
  margin-top: -18px;  /* provide offset for in-page anchors */
  padding-top: 24px;
 }
 h4 {
  margin-top: -18px;  /* provide offset for in-page anchors */
  padding-top: 24px;
 }
/* Float artwork pilcrow to the right */
@media screen {
  .artwork a.pilcrow {
    display: block;
    line-height: 0.7;
    margin-top: 0.15em;
  }
}
/* Make pilcrows on dd visible */
@media screen {
  dd:hover > a.pilcrow {
    visibility: visible;
  }
}
/* Make the placement of figcaption match that of a table's caption
   by removing the figure's added bottom margin */
.alignLeft.art-text,
.alignCenter.art-text,
.alignRight.art-text {
   margin-bottom: 0;
}
.alignLeft,
.alignCenter,
.alignRight {
  margin: 1em 0 0 0;
}
/* In print, the pilcrow won't show on hover, so prevent it from taking up space,
   possibly even requiring a new line */
@media print {
  a.pilcrow {
    display: none;
  }
}
/* Styling for the external metadata */
div#external-metadata {
  background-color: #eee;
  padding: 0.5em;
  margin-bottom: 0.5em;
  display: none;
}
div#internal-metadata {
  padding: 0.5em;                       /* to match the external-metadata padding */
}
/* Styling for title RFC Number */
h1#rfcnum {
  clear: both;
  margin: 0 0 -1em;
  padding: 1em 0 0 0;
}
/* Make .olPercent look the same as <ol><li> */
dl.olPercent > dd {
  margin-bottom: 0.25em;
  min-height: initial;
}
/* Give aside some styling to set it apart */
aside {
  border-left: 1px solid #ddd;
  margin: 1em 0 1em 2em;
  padding: 0.2em 2em;
}
aside > dl,
aside > ol,
aside > ul,
aside > table,
aside > p {
  margin-bottom: 0.5em;
}
/* Additional page break settings */
@media print {
  figcaption, table caption {
    page-break-before: avoid;
  }
}
/* Font size adjustments for print */
@media print {
  body  { font-size: 10pt;      line-height: normal; max-width: 96%; }
  h1    { font-size: 1.72em;    padding-top: 1.5em; } /* 1*1.2*1.2*1.2 */
  h2    { font-size: 1.44em;    padding-top: 1.5em; } /* 1*1.2*1.2 */
  h3    { font-size: 1.2em;     padding-top: 1.5em; } /* 1*1.2 */
  h4    { font-size: 1em;       padding-top: 1.5em; }
  h5, h6 { font-size: 1em;      margin: initial; padding: 0.5em 0 0.3em; }
}
/* Sourcecode margin in print, when there's no pilcrow */
@media print {
  .artwork,
  .sourcecode {
    margin-bottom: 1em;
  }
}
/* Avoid narrow tables forcing too narrow table captions, which may render badly */
table {
  min-width: 20em;
}
/* ol type a */
ol.type-a { list-style-type: lower-alpha; }
ol.type-A { list-style-type: upper-alpha; }
ol.type-i { list-style-type: lower-roman; }
ol.type-I { list-style-type: lower-roman; }
/* Apply the print table and row borders in general, on request from the RPC,
and increase the contrast between border and odd row background sligthtly */
table {
  border: 1px solid #ddd;
}
td {
  border-top: 1px solid #ddd;
}
tr {
  break-inside: avoid;
}
tr:nth-child(2n+1) > td {
  background-color: #f8f8f8;
}
/* Use style rules to govern display of the TOC. */
@media screen and (max-width: 1023px) {
  #toc nav { display: none; }
  #toc.active nav { display: block; }
}
/* Add support for keepWithNext */
.keepWithNext {
  break-after: avoid-page;
  break-after: avoid-page;
}
/* Add support for keepWithPrevious */
.keepWithPrevious {
  break-before: avoid-page;
}
/* Change the approach to avoiding breaks inside artwork etc. */
figure, pre, table, .artwork, .sourcecode  {
  break-before: auto;
  break-after: auto;
}
/* Avoid breaks between <dt> and <dd> */
dl {
  break-before: auto;
  break-inside: auto;
}
dt {
  break-before: auto;
  break-after: avoid-page;
}
dd {
  break-before: avoid-page;
  break-after: auto;
  orphans: 3;
  widows: 3
}
span.break, dd.break {
  margin-bottom: 0;
  min-height: 0;
  break-before: auto;
  break-inside: auto;
  break-after: auto;
}
/* Undo break-before ToC */
@media print {
  #toc {
    break-before: auto;
  }
}
/* Text in compact lists should not get extra bottim margin space,
   since that would makes the list not compact */
ul.compact p, .ulCompact p,
ol.compact p, .olCompact p {
 margin: 0;
}
/* But the list as a whole needs the extra space at the end */
section ul.compact,
section .ulCompact,
section ol.compact,
section .olCompact {
  margin-bottom: 1em;                    /* same as p not within ul.compact etc. */
}
/* The tt and code background above interferes with for instance table cell
   backgrounds.  Changed to something a bit more selective. */
tt, code {
  background-color: transparent;
}
p tt, p code, li tt, li code {
  background-color: #f8f8f8;
}
/* Tweak the pre margin -- 0px doesn't come out well */
pre {
   margin-top: 0.5px;
}
/* Tweak the comact list text */
ul.compact, .ulCompact,
ol.compact, .olCompact,
dl.compact, .dlCompact {
  line-height: normal;
}
/* Don't add top margin for nested lists */
li > ul, li > ol, li > dl,
dd > ul, dd > ol, dd > dl,
dl > dd > dl {
  margin-top: initial;
}
/* Elements that should not be rendered on the same line as a <dt> */
/* This should match the element list in writer.text.TextWriter.render_dl() */
dd > div.artwork:first-child,
dd > aside:first-child,
dd > figure:first-child,
dd > ol:first-child,
dd > div:first-child > pre.sourcecode,
dd > table:first-child,
dd > ul:first-child {
  clear: left;
}
/* fix for weird browser behaviour when <dd/> is empty */
dt+dd:empty::before{
  content: "\00a0";
}
/* Make paragraph spacing inside <li> smaller than in body text, to fit better within the list */
li > p {
  margin-bottom: 0.5em
}
/* Don't let p margin spill out from inside list items */
li > p:last-of-type {
  margin-bottom: 0;
}
</style>
<link href="rfc-local.css" rel="stylesheet" type="text/css">
<link href="https://dx.doi.org/10.17487/rfc9309" rel="alternate">
  <link href="urn:issn:2070-1721" rel="alternate">
  <link href="https://datatracker.ietf.org/doc/draft-koster-rep-12" rel="prev">
  </head>
<body>
<script src="https://www.rfc-editor.org/js/metadata.min.js"></script>
<table class="ears">
<thead><tr>
<td class="left">RFC 9309</td>
<td class="center">Robots Exclusion Protocol (REP)</td>
<td class="right">September 2022</td>
</tr></thead>
<tfoot><tr>
<td class="left">Koster, et al.</td>
<td class="center">Standards Track</td>
<td class="right">[Page]</td>
</tr></tfoot>
</table>
<div id="external-metadata" class="document-information"></div>
<div id="internal-metadata" class="document-information">
<dl id="identifiers">
<dt class="label-stream">Stream:</dt>
<dd class="stream">Internet Engineering Task Force (IETF)</dd>
<dt class="label-rfc">RFC:</dt>
<dd class="rfc"><a href="https://www.rfc-editor.org/rfc/rfc9309" class="eref">9309</a></dd>
<dt class="label-category">Category:</dt>
<dd class="category">Standards Track</dd>
<dt class="label-published">Published:</dt>
<dd class="published">
<time datetime="2022-09" class="published">September 2022</time>
    </dd>
<dt class="label-issn">ISSN:</dt>
<dd class="issn">2070-1721</dd>
<dt class="label-authors">Authors:</dt>
<dd class="authors">
<div class="author">
      <div class="author-name">M. Koster</div>
</div>
<div class="author">
      <div class="author-name">G. Illyes</div>
<div class="org">Google LLC</div>
</div>
<div class="author">
      <div class="author-name">H. Zeller</div>
<div class="org">Google LLC</div>
</div>
<div class="author">
      <div class="author-name">L. Sassman</div>
<div class="org">Google LLC</div>
</div>
</dd>
</dl>
</div>
<h1 id="rfcnum">RFC 9309</h1>
<h1 id="title">Robots Exclusion Protocol</h1>
<section id="section-abstract">
      <h2 id="abstract"><a href="#abstract" class="selfRef">Abstract</a></h2>
<p id="section-abstract-1"> This document specifies and extends the "Robots Exclusion Protocol"
          method originally defined by Martijn Koster in 1994 for service owners
          to control how content served by their services may be accessed, if at
          all, by automatic clients known as crawlers. Specifically, it adds
          definition language for the protocol, instructions for handling
          errors, and instructions for caching.<a href="#section-abstract-1" class="pilcrow">¶</a></p>
</section>
<div id="status-of-memo">
<section id="section-boilerplate.1">
        <h2 id="name-status-of-this-memo">
<a href="#name-status-of-this-memo" class="section-name selfRef">Status of This Memo</a>
        </h2>
<p id="section-boilerplate.1-1">
            This is an Internet Standards Track document.<a href="#section-boilerplate.1-1" class="pilcrow">¶</a></p>
<p id="section-boilerplate.1-2">
            This document is a product of the Internet Engineering Task Force
            (IETF).  It represents the consensus of the IETF community.  It has
            received public review and has been approved for publication by
            the Internet Engineering Steering Group (IESG).  Further
            information on Internet Standards is available in Section 2 of 
            RFC 7841.<a href="#section-boilerplate.1-2" class="pilcrow">¶</a></p>
<p id="section-boilerplate.1-3">
            Information about the current status of this document, any
            errata, and how to provide feedback on it may be obtained at
            <span><a href="https://www.rfc-editor.org/info/rfc9309">https://www.rfc-editor.org/info/rfc9309</a></span>.<a href="#section-boilerplate.1-3" class="pilcrow">¶</a></p>
</section>
</div>
<div id="copyright">
<section id="section-boilerplate.2">
        <h2 id="name-copyright-notice">
<a href="#name-copyright-notice" class="section-name selfRef">Copyright Notice</a>
        </h2>
<p id="section-boilerplate.2-1">
            Copyright (c) 2022 IETF Trust and the persons identified as the
            document authors. All rights reserved.<a href="#section-boilerplate.2-1" class="pilcrow">¶</a></p>
<p id="section-boilerplate.2-2">
            This document is subject to BCP 78 and the IETF Trust's Legal
            Provisions Relating to IETF Documents
            (<span><a href="https://trustee.ietf.org/license-info">https://trustee.ietf.org/license-info</a></span>) in effect on the date of
            publication of this document. Please review these documents
            carefully, as they describe your rights and restrictions with
            respect to this document. Code Components extracted from this
            document must include Revised BSD License text as described in
            Section 4.e of the Trust Legal Provisions and are provided without
            warranty as described in the Revised BSD License.<a href="#section-boilerplate.2-2" class="pilcrow">¶</a></p>
</section>
</div>
<div id="toc">
<section id="section-toc.1">
        <a href="#" onclick="scroll(0,0)" class="toplink">▲</a><h2 id="name-table-of-contents">
<a href="#name-table-of-contents" class="section-name selfRef">Table of Contents</a>
        </h2>
<nav class="toc"><ul class="compact toc ulBare ulEmpty">
<li class="compact toc ulBare ulEmpty" id="section-toc.1-1.1">
            <p id="section-toc.1-1.1.1" class="keepWithNext"><a href="#section-1" class="xref">1</a>.  <a href="#name-introduction" class="xref">Introduction</a></p>
<ul class="compact toc ulBare ulEmpty">
<li class="compact toc ulBare ulEmpty" id="section-toc.1-1.1.2.1">
                <p id="section-toc.1-1.1.2.1.1" class="keepWithNext"><a href="#section-1.1" class="xref">1.1</a>.  <a href="#name-requirements-language" class="xref">Requirements Language</a></p>
</li>
            </ul>
</li>
          <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2">
            <p id="section-toc.1-1.2.1"><a href="#section-2" class="xref">2</a>.  <a href="#name-specification" class="xref">Specification</a></p>
<ul class="compact toc ulBare ulEmpty">
<li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.1">
                <p id="section-toc.1-1.2.2.1.1" class="keepWithNext"><a href="#section-2.1" class="xref">2.1</a>.  <a href="#name-protocol-definition" class="xref">Protocol Definition</a></p>
</li>
              <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.2">
                <p id="section-toc.1-1.2.2.2.1"><a href="#section-2.2" class="xref">2.2</a>.  <a href="#name-formal-syntax" class="xref">Formal Syntax</a></p>
<ul class="compact toc ulBare ulEmpty">
<li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.2.2.1">
                    <p id="section-toc.1-1.2.2.2.2.1.1"><a href="#section-2.2.1" class="xref">2.2.1</a>.  <a href="#name-the-user-agent-line" class="xref">The User-Agent Line</a></p>
</li>
                  <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.2.2.2">
                    <p id="section-toc.1-1.2.2.2.2.2.1"><a href="#section-2.2.2" class="xref">2.2.2</a>.  <a href="#name-the-allow-and-disallow-line" class="xref">The "Allow" and "Disallow" Lines</a></p>
</li>
                  <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.2.2.3">
                    <p id="section-toc.1-1.2.2.2.2.3.1"><a href="#section-2.2.3" class="xref">2.2.3</a>.  <a href="#name-special-characters" class="xref">Special Characters</a></p>
</li>
                  <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.2.2.4">
                    <p id="section-toc.1-1.2.2.2.2.4.1"><a href="#section-2.2.4" class="xref">2.2.4</a>.  <a href="#name-other-records" class="xref">Other Records</a></p>
</li>
                </ul>
</li>
              <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.3">
                <p id="section-toc.1-1.2.2.3.1"><a href="#section-2.3" class="xref">2.3</a>.  <a href="#name-access-method" class="xref">Access Method</a></p>
<ul class="compact toc ulBare ulEmpty">
<li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.3.2.1">
                    <p id="section-toc.1-1.2.2.3.2.1.1"><a href="#section-2.3.1" class="xref">2.3.1</a>.  <a href="#name-access-results" class="xref">Access Results</a></p>
<ul class="compact toc ulBare ulEmpty">
<li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.3.2.1.2.1">
                        <p id="section-toc.1-1.2.2.3.2.1.2.1.1"><a href="#section-2.3.1.1" class="xref">2.3.1.1</a>.  <a href="#name-successful-access" class="xref">Successful Access</a></p>
</li>
                      <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.3.2.1.2.2">
                        <p id="section-toc.1-1.2.2.3.2.1.2.2.1"><a href="#section-2.3.1.2" class="xref">2.3.1.2</a>.  <a href="#name-redirects" class="xref">Redirects</a></p>
</li>
                      <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.3.2.1.2.3">
                        <p id="section-toc.1-1.2.2.3.2.1.2.3.1"><a href="#section-2.3.1.3" class="xref">2.3.1.3</a>.  <a href="#name-unavailable-status" class="xref">"Unavailable" Status</a></p>
</li>
                      <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.3.2.1.2.4">
                        <p id="section-toc.1-1.2.2.3.2.1.2.4.1"><a href="#section-2.3.1.4" class="xref">2.3.1.4</a>.  <a href="#name-unreachable-status" class="xref">"Unreachable" Status</a></p>
</li>
                      <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.3.2.1.2.5">
                        <p id="section-toc.1-1.2.2.3.2.1.2.5.1"><a href="#section-2.3.1.5" class="xref">2.3.1.5</a>.  <a href="#name-parsing-errors" class="xref">Parsing Errors</a></p>
</li>
                    </ul>
</li>
                </ul>
</li>
              <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.4">
                <p id="section-toc.1-1.2.2.4.1"><a href="#section-2.4" class="xref">2.4</a>.  <a href="#name-caching" class="xref">Caching</a></p>
</li>
              <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.2.2.5">
                <p id="section-toc.1-1.2.2.5.1"><a href="#section-2.5" class="xref">2.5</a>.  <a href="#name-limits" class="xref">Limits</a></p>
</li>
            </ul>
</li>
          <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.3">
            <p id="section-toc.1-1.3.1"><a href="#section-3" class="xref">3</a>.  <a href="#name-security-considerations" class="xref">Security Considerations</a></p>
</li>
          <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.4">
            <p id="section-toc.1-1.4.1"><a href="#section-4" class="xref">4</a>.  <a href="#name-iana-considerations" class="xref">IANA Considerations</a></p>
</li>
          <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.5">
            <p id="section-toc.1-1.5.1"><a href="#section-5" class="xref">5</a>.  <a href="#name-examples" class="xref">Examples</a></p>
<ul class="compact toc ulBare ulEmpty">
<li class="compact toc ulBare ulEmpty" id="section-toc.1-1.5.2.1">
                <p id="section-toc.1-1.5.2.1.1"><a href="#section-5.1" class="xref">5.1</a>.  <a href="#name-simple-example" class="xref">Simple Example</a></p>
</li>
              <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.5.2.2">
                <p id="section-toc.1-1.5.2.2.1"><a href="#section-5.2" class="xref">5.2</a>.  <a href="#name-longest-match" class="xref">Longest Match</a></p>
</li>
            </ul>
</li>
          <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.6">
            <p id="section-toc.1-1.6.1"><a href="#section-6" class="xref">6</a>.  <a href="#name-references" class="xref">References</a></p>
<ul class="compact toc ulBare ulEmpty">
<li class="compact toc ulBare ulEmpty" id="section-toc.1-1.6.2.1">
                <p id="section-toc.1-1.6.2.1.1"><a href="#section-6.1" class="xref">6.1</a>.  <a href="#name-normative-references" class="xref">Normative References</a></p>
</li>
              <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.6.2.2">
                <p id="section-toc.1-1.6.2.2.1"><a href="#section-6.2" class="xref">6.2</a>.  <a href="#name-informative-references" class="xref">Informative References</a></p>
</li>
            </ul>
</li>
          <li class="compact toc ulBare ulEmpty" id="section-toc.1-1.7">
            <p id="section-toc.1-1.7.1"><a href="#appendix-A" class="xref"></a><a href="#name-authors-addresses" class="xref">Authors' Addresses</a></p>
</li>
        </ul>
</nav>
</section>
</div>
<div id="introduction">
<section id="section-1">
      <h2 id="name-introduction">
<a href="#section-1" class="section-number selfRef">1. </a><a href="#name-introduction" class="section-name selfRef">Introduction</a>
      </h2>
<p id="section-1-1"> This document applies to services that provide resources that clients
          can access through URIs as defined in <span>[<a href="#RFC3986" class="xref">RFC3986</a>]</span>. For example,
          in the context of HTTP, a browser is a client that displays the content of a
          web page.<a href="#section-1-1" class="pilcrow">¶</a></p>
<p id="section-1-2"> Crawlers are automated clients. Search engines, for instance, have crawlers to
          recursively traverse links for indexing as defined in
          <span>[<a href="#RFC8288" class="xref">RFC8288</a>]</span>.<a href="#section-1-2" class="pilcrow">¶</a></p>
<p id="section-1-3"> It may be inconvenient for service owners if crawlers visit the entirety of
          their URI space. This document specifies the rules originally defined by
          the "Robots Exclusion Protocol" <span>[<a href="#ROBOTSTXT" class="xref">ROBOTSTXT</a>]</span> that crawlers
          are requested to honor when accessing URIs.<a href="#section-1-3" class="pilcrow">¶</a></p>
<p id="section-1-4"> These rules are not a form of access authorization.<a href="#section-1-4" class="pilcrow">¶</a></p>
<div id="requirements-language">
<section id="section-1.1">
        <h3 id="name-requirements-language">
<a href="#section-1.1" class="section-number selfRef">1.1. </a><a href="#name-requirements-language" class="section-name selfRef">Requirements Language</a>
        </h3>
<p id="section-1.1-1">The key words "<span class="bcp14">MUST</span>", "<span class="bcp14">MUST NOT</span>",
        "<span class="bcp14">REQUIRED</span>", "<span class="bcp14">SHALL</span>",
        "<span class="bcp14">SHALL NOT</span>", "<span class="bcp14">SHOULD</span>",
        "<span class="bcp14">SHOULD NOT</span>",
        "<span class="bcp14">RECOMMENDED</span>", "<span class="bcp14">NOT RECOMMENDED</span>",
        "<span class="bcp14">MAY</span>", and "<span class="bcp14">OPTIONAL</span>" in this document
        are to be interpreted as described in BCP 14
        <span>[<a href="#RFC2119" class="xref">RFC2119</a>]</span> <span>[<a href="#RFC8174" class="xref">RFC8174</a>]</span> when, and only
        when, they appear in all capitals, as shown here.<a href="#section-1.1-1" class="pilcrow">¶</a></p>
</section>
</div>
</section>
</div>
<div id="specification">
<section id="section-2">
      <h2 id="name-specification">
<a href="#section-2" class="section-number selfRef">2. </a><a href="#name-specification" class="section-name selfRef">Specification</a>
      </h2>
<div id="protocol-definition">
<section id="section-2.1">
        <h3 id="name-protocol-definition">
<a href="#section-2.1" class="section-number selfRef">2.1. </a><a href="#name-protocol-definition" class="section-name selfRef">Protocol Definition</a>
        </h3>
<p id="section-2.1-1"> The protocol language consists of rule(s) and group(s) that the service
            makes available in a file named "robots.txt" as described in
            <a href="#access-method" class="xref">Section 2.3</a>:<a href="#section-2.1-1" class="pilcrow">¶</a></p>
<span class="break"></span><dl class="dlParallel" id="section-2.1-2">
          <dt id="section-2.1-2.1"> Rule:</dt>
          <dd style="margin-left: 1.5em" id="section-2.1-2.2"> A line with a key-value pair that defines how a
                crawler may access URIs. See
                <a href="#the-allow-and-disallow-lines" class="xref">Section 2.2.2</a>.<a href="#section-2.1-2.2" class="pilcrow">¶</a>
</dd>
          <dd class="break"></dd>
<dt id="section-2.1-2.3"> Group:</dt>
          <dd style="margin-left: 1.5em" id="section-2.1-2.4"> One or more user-agent lines that are followed by
                one or more rules. The group is terminated by a user-agent line
                or end of file. See <a href="#the-user-agent-line" class="xref">Section 2.2.1</a>.
                The last group may have no rules, which means it implicitly
                allows everything.<a href="#section-2.1-2.4" class="pilcrow">¶</a>
</dd>
        <dd class="break"></dd>
</dl>
</section>
</div>
<div id="formal-syntax">
<section id="section-2.2">
        <h3 id="name-formal-syntax">
<a href="#section-2.2" class="section-number selfRef">2.2. </a><a href="#name-formal-syntax" class="section-name selfRef">Formal Syntax</a>
        </h3>
<p id="section-2.2-1"> Below is an Augmented Backus-Naur Form (ABNF) description, as described
            in <span>[<a href="#RFC5234" class="xref">RFC5234</a>]</span>.<a href="#section-2.2-1" class="pilcrow">¶</a></p>
<div id="section-2.2-2">
<pre class="lang-abnf sourcecode">
 robotstxt = *(group / emptyline)
 group = startgroupline                ; We start with a user-agent
                                       ; line
        *(startgroupline / emptyline)  ; ... and possibly more
                                       ; user-agent lines
        *(rule / emptyline)            ; followed by rules relevant
                                       ; for the preceding
                                       ; user-agent lines

 startgroupline = *WS "user-agent" *WS ":" *WS product-token EOL

 rule = *WS ("allow" / "disallow") *WS ":"
       *WS (path-pattern / empty-pattern) EOL

 ; parser implementors: define additional lines you need (for
 ; example, Sitemaps).

 product-token = identifier / "*"
 path-pattern = "/" *UTF8-char-noctl ; valid URI path pattern
 empty-pattern = *WS

 identifier = 1*(%x2D / %x41-5A / %x5F / %x61-7A)
 comment = "#" *(UTF8-char-noctl / WS / "#")
 emptyline = EOL
 EOL = *WS [comment] NL ; end-of-line may have
                        ; optional trailing comment
 NL = %x0D / %x0A / %x0D.0A
 WS = %x20 / %x09

 ; UTF8 derived from RFC 3629, but excluding control characters

 UTF8-char-noctl = UTF8-1-noctl / UTF8-2 / UTF8-3 / UTF8-4
 UTF8-1-noctl = %x21 / %x22 / %x24-7F ; excluding control, space, "#"
 UTF8-2 = %xC2-DF UTF8-tail
 UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2UTF8-tail /
          %xED %x80-9F UTF8-tail / %xEE-EF 2UTF8-tail
 UTF8-4 = %xF0 %x90-BF 2UTF8-tail / %xF1-F3 3UTF8-tail /
          %xF4 %x80-8F 2UTF8-tail

 UTF8-tail = %x80-BF
</pre><a href="#section-2.2-2" class="pilcrow">¶</a>
</div>
<div id="the-user-agent-line">
<section id="section-2.2.1">
          <h4 id="name-the-user-agent-line">
<a href="#section-2.2.1" class="section-number selfRef">2.2.1. </a><a href="#name-the-user-agent-line" class="section-name selfRef">The User-Agent Line</a>
          </h4>
<p id="section-2.2.1-1"> Crawlers set their own name, which is called a product token, to find
              relevant groups. The product token <span class="bcp14">MUST</span> contain only
              uppercase and lowercase letters ("a-z" and "A-Z"),
              underscores ("_"), and hyphens ("-").
              The product token <span class="bcp14">SHOULD</span>
              be a substring of the identification string that the crawler sends to
              the service. For example, in the case of HTTP
              <span>[<a href="#RFC9110" class="xref">RFC9110</a>]</span>, the product token
              <span class="bcp14">SHOULD</span> be a substring in the User-Agent header.
              The identification string <span class="bcp14">SHOULD</span> describe the purpose of
              the crawler. Here's an example of a User-Agent HTTP request header
              with a link pointing to a page describing the purpose of the
              ExampleBot crawler, which appears as a substring in the User-Agent HTTP
              header and as a product token in the robots.txt user-agent line:<a href="#section-2.2.1-1" class="pilcrow">¶</a></p>
<span id="name-example-of-a-user-agent-htt"></span><div id="fig-1">
<figure id="figure-1">
            <div class="alignCenter art-text artwork" id="section-2.2.1-2.1">
<pre>
+==========================================+========================+
| User-Agent HTTP header                   | robots.txt user-agent  |
|                                          | line                   |
+==========================================+========================+
| User-Agent: Mozilla/5.0 (compatible;     | user-agent: ExampleBot |
| ExampleBot/0.1;                          |                        |
| https://www.example.com/bot.html)        |                        |
+------------------------------------------+------------------------+
</pre>
</div>
<figcaption><a href="#figure-1" class="selfRef">Figure 1</a>:
<a href="#name-example-of-a-user-agent-htt" class="selfRef">Example of a User-Agent HTTP header and
                  robots.txt user-agent line for the ExampleBot product token</a>
            </figcaption></figure>
</div>
<p id="section-2.2.1-3"> Note that the product token (ExampleBot) is a substring of
            the User-Agent HTTP header.<a href="#section-2.2.1-3" class="pilcrow">¶</a></p>
<p id="section-2.2.1-4"> Crawlers <span class="bcp14">MUST</span> use case-insensitive matching
              to find the group that matches the product token and then
              obey the rules of the group. If there is more than one
              group matching the user-agent, the matching groups' rules
              <span class="bcp14">MUST</span> be combined into one group and parsed
              according to
              <a href="#the-allow-and-disallow-lines" class="xref">Section 2.2.2</a>.<a href="#section-2.2.1-4" class="pilcrow">¶</a></p>
<span id="name-example-of-how-to-merge-two"></span><div id="fig-2">
<figure id="figure-2">
            <div class="alignCenter art-text artwork" id="section-2.2.1-5.1">
<pre>
+========================================+========================+
| Two groups that match the same product | Merged group           |
| token exactly                          |                        |
+========================================+========================+
| user-agent: ExampleBot                 | user-agent: ExampleBot |
| disallow: /foo                         | disallow: /foo         |
| disallow: /bar                         | disallow: /bar         |
|                                        | disallow: /baz         |
| user-agent: ExampleBot                 |                        |
| disallow: /baz                         |                        |
+----------------------------------------+------------------------+
</pre>
</div>
<figcaption><a href="#figure-2" class="selfRef">Figure 2</a>:
<a href="#name-example-of-how-to-merge-two" class="selfRef">Example of how to merge two robots.txt
                  groups that match the same product token</a>
            </figcaption></figure>
</div>
<p id="section-2.2.1-6"> If no matching group exists, crawlers <span class="bcp14">MUST</span> obey the group
              with a user-agent line with the "*" value, if present.<a href="#section-2.2.1-6" class="pilcrow">¶</a></p>
<span id="name-example-of-no-matching-grou"></span><div id="fig-3">
<figure id="figure-3">
            <div class="alignCenter art-text artwork" id="section-2.2.1-7.1">
<pre>
+==================================+======================+
| Two groups that don't explicitly | Applicable group for |
| match ExampleBot                 | ExampleBot           |
+==================================+======================+
| user-agent: *                    | user-agent: *        |
| disallow: /foo                   | disallow: /foo       |
| disallow: /bar                   | disallow: /bar       |
|                                  |                      |
| user-agent: BazBot               |                      |
| disallow: /baz                   |                      |
+----------------------------------+----------------------+
</pre>
</div>
<figcaption><a href="#figure-3" class="selfRef">Figure 3</a>:
<a href="#name-example-of-no-matching-grou" class="selfRef">Example of no matching groups other than the "*"
                  for the ExampleBot product token</a>
            </figcaption></figure>
</div>
<p id="section-2.2.1-8"> If no group matches the product token and there is no group with a user-agent
              line with the "*" value, or no groups are present at all, no
              rules apply.<a href="#section-2.2.1-8" class="pilcrow">¶</a></p>
</section>
</div>
<div id="the-allow-and-disallow-lines">
<section id="section-2.2.2">
          <h4 id="name-the-allow-and-disallow-line">
<a href="#section-2.2.2" class="section-number selfRef">2.2.2. </a><a href="#name-the-allow-and-disallow-line" class="section-name selfRef">The "Allow" and "Disallow" Lines</a>
          </h4>
<p id="section-2.2.2-1"> These lines indicate whether accessing a URI that matches the
              corresponding path is allowed or disallowed.<a href="#section-2.2.2-1" class="pilcrow">¶</a></p>
<p id="section-2.2.2-2"> To evaluate if access to a URI is allowed, a crawler <span class="bcp14">MUST</span>
              match the paths in "allow" and "disallow" rules against the URI.
              The matching <span class="bcp14">SHOULD</span> be case sensitive. The matching
              <span class="bcp14">MUST</span> start with the first octet of the path. The most
              specific match found <span class="bcp14">MUST</span> be used. The most specific
              match is the match that has the most octets. Duplicate rules in a
              group <span class="bcp14">MAY</span> be deduplicated. If an "allow" rule and a "disallow"
              rule are equivalent, then the "allow" rule <span class="bcp14">SHOULD</span> be used. If no
              match is found amongst the rules in a group for a matching user-agent
              or there are no rules in the group, the URI is allowed. The
              /robots.txt URI is implicitly allowed.<a href="#section-2.2.2-2" class="pilcrow">¶</a></p>
<p id="section-2.2.2-3"> Octets in the URI and robots.txt paths outside the range of the
              ASCII coded character set, and those in the reserved range defined
              by <span>[<a href="#RFC3986" class="xref">RFC3986</a>]</span>, <span class="bcp14">MUST</span> be percent-encoded as
              defined by <span>[<a href="#RFC3986" class="xref">RFC3986</a>]</span> prior to comparison.<a href="#section-2.2.2-3" class="pilcrow">¶</a></p>
<p id="section-2.2.2-4"> If a percent-encoded ASCII octet is encountered in the URI, it
              <span class="bcp14">MUST</span> be unencoded prior to comparison, unless it is a
              reserved character in the URI as defined by <span>[<a href="#RFC3986" class="xref">RFC3986</a>]</span>
              or the character is outside the unreserved character range. The match
              evaluates positively if and only if the end of the path from the rule
              is reached before a difference in octets is encountered.<a href="#section-2.2.2-4" class="pilcrow">¶</a></p>
<p id="section-2.2.2-5"> For example:<a href="#section-2.2.2-5" class="pilcrow">¶</a></p>
<span id="name-examples-of-matching-percen"></span><div id="fig-4">
<figure id="figure-4">
            <div class="alignCenter art-text artwork" id="section-2.2.2-6.1">
<pre>
+==================+=======================+=======================+
| Path             | Encoded Path          | Path to Match         |
+==================+=======================+=======================+
| /foo/bar?baz=quz | /foo/bar?baz=quz      | /foo/bar?baz=quz      |
+------------------+-----------------------+-----------------------+
| /foo/bar?baz=    | /foo/bar?baz=         | /foo/bar?baz=         |
| https://foo.bar  | https%3A%2F%2Ffoo.bar | https%3A%2F%2Ffoo.bar |
+------------------+-----------------------+-----------------------+
| /foo/bar/        | /foo/bar/%E3%83%84    | /foo/bar/%E3%83%84    |
| U+E38384         |                       |                       |
+------------------+-----------------------+-----------------------+
| /foo/            | /foo/bar/%E3%83%84    | /foo/bar/%E3%83%84    |
| bar/%E3%83%84    |                       |                       |
+------------------+-----------------------+-----------------------+
| /foo/            | /foo/bar/%62%61%7A    | /foo/bar/baz          |
| bar/%62%61%7A    |                       |                       |
+------------------+-----------------------+-----------------------+
</pre>
</div>
<figcaption><a href="#figure-4" class="selfRef">Figure 4</a>:
<a href="#name-examples-of-matching-percen" class="selfRef">Examples of matching percent-encoded URI components</a>
            </figcaption></figure>
</div>
<p id="section-2.2.2-7"> The crawler <span class="bcp14">SHOULD</span> ignore "disallow" and
              "allow" rules that are not in any group (for example, any
              rule that precedes the first user-agent line).<a href="#section-2.2.2-7" class="pilcrow">¶</a></p>
<p id="section-2.2.2-8"> Implementors <span class="bcp14">MAY</span> bridge encoding mismatches if they
              detect that the robots.txt file is not UTF-8 encoded.<a href="#section-2.2.2-8" class="pilcrow">¶</a></p>
</section>
</div>
<div id="special-characters">
<section id="section-2.2.3">
          <h4 id="name-special-characters">
<a href="#section-2.2.3" class="section-number selfRef">2.2.3. </a><a href="#name-special-characters" class="section-name selfRef">Special Characters</a>
          </h4>
<p id="section-2.2.3-1"> Crawlers <span class="bcp14">MUST</span> support the following special characters:<a href="#section-2.2.3-1" class="pilcrow">¶</a></p>
<span id="name-list-of-special-characters-"></span><div id="fig-5">
<figure id="figure-5">
            <div class="alignCenter art-text artwork" id="section-2.2.3-2.1">
<pre>
+===========+===================+==============================+
| Character | Description       | Example                      |
+===========+===================+==============================+
| #         | Designates a line | allow: / # comment in line   |
|           | comment.          |                              |
|           |                   | # comment on its own line    |
+-----------+-------------------+------------------------------+
| $         | Designates the    | allow: /this/path/exactly$   |
|           | end of the match  |                              |
|           | pattern.          |                              |
+-----------+-------------------+------------------------------+
| *         | Designates 0 or   | allow: /this/*/exactly       |
|           | more instances of |                              |
|           | any character.    |                              |
+-----------+-------------------+------------------------------+
</pre>
</div>
<figcaption><a href="#figure-5" class="selfRef">Figure 5</a>:
<a href="#name-list-of-special-characters-" class="selfRef">List of special characters in robots.txt files</a>
            </figcaption></figure>
</div>
<p id="section-2.2.3-3"> If crawlers match special characters verbatim in the URI, crawlers
              <span class="bcp14">SHOULD</span> use "%" encoding. For example:<a href="#section-2.2.3-3" class="pilcrow">¶</a></p>
<span id="name-example-of-percent-encoding"></span><div id="fig-6">
<figure id="figure-6">
            <div class="alignCenter art-text artwork" id="section-2.2.3-4.1">
<pre>
+============================+====================================+
| Percent-encoded Pattern    | URI                                |
+============================+====================================+
| /path/file-with-a-%2A.html | https://www.example.com/path/      |
|                            | file-with-a-*.html                 |
+----------------------------+------------------------------------+
| /path/foo-%24              | https://www.example.com/path/foo-$ |
+----------------------------+------------------------------------+
</pre>
</div>
<figcaption><a href="#figure-6" class="selfRef">Figure 6</a>:
<a href="#name-example-of-percent-encoding" class="selfRef">Example of percent-encoding</a>
            </figcaption></figure>
</div>
</section>
</div>
<div id="other-records">
<section id="section-2.2.4">
          <h4 id="name-other-records">
<a href="#section-2.2.4" class="section-number selfRef">2.2.4. </a><a href="#name-other-records" class="section-name selfRef">Other Records</a>
          </h4>
<p id="section-2.2.4-1"> Crawlers <span class="bcp14">MAY</span> interpret other records that are not
              part of the robots.txt protocol -- for example, "Sitemaps"
              <span>[<a href="#SITEMAPS" class="xref">SITEMAPS</a>]</span>. Crawlers <span class="bcp14">MAY</span> be lenient when
              interpreting other records. For example, crawlers may accept
              common misspellings of the record.<a href="#section-2.2.4-1" class="pilcrow">¶</a></p>
<p id="section-2.2.4-2"> Parsing of other records
              <span class="bcp14">MUST NOT</span> interfere with the parsing of explicitly
              defined records in <a href="#specification" class="xref">Section 2</a>.
              For example, a "Sitemaps" record <span class="bcp14">MUST NOT</span> terminate a
              group.<a href="#section-2.2.4-2" class="pilcrow">¶</a></p>
</section>
</div>
</section>
</div>
<div id="access-method">
<section id="section-2.3">
        <h3 id="name-access-method">
<a href="#section-2.3" class="section-number selfRef">2.3. </a><a href="#name-access-method" class="section-name selfRef">Access Method</a>
        </h3>
<p id="section-2.3-1"> The rules <span class="bcp14">MUST</span> be accessible in a file named
          "/robots.txt" (all lowercase) in the top-level path of
          the service. The file <span class="bcp14">MUST</span> be UTF-8 encoded (as
          defined in <span>[<a href="#RFC3629" class="xref">RFC3629</a>]</span>) and Internet Media Type
          "text/plain"
          (as defined in <span>[<a href="#RFC2046" class="xref">RFC2046</a>]</span>).<a href="#section-2.3-1" class="pilcrow">¶</a></p>
<p id="section-2.3-2"> As per <span>[<a href="#RFC3986" class="xref">RFC3986</a>]</span>, the URI of the robots.txt file is:<a href="#section-2.3-2" class="pilcrow">¶</a></p>
<p id="section-2.3-3"> "scheme:[//authority]/robots.txt"<a href="#section-2.3-3" class="pilcrow">¶</a></p>
<p id="section-2.3-4"> For example, in the context of HTTP or FTP, the URI is:<a href="#section-2.3-4" class="pilcrow">¶</a></p>
<div class="alignLeft art-text artwork" id="section-2.3-5">
<pre>
          https://www.example.com/robots.txt

          ftp://ftp.example.com/robots.txt
</pre><a href="#section-2.3-5" class="pilcrow">¶</a>
</div>
<div id="access-results">
<section id="section-2.3.1">
          <h4 id="name-access-results">
<a href="#section-2.3.1" class="section-number selfRef">2.3.1. </a><a href="#name-access-results" class="section-name selfRef">Access Results</a>
          </h4>
<div id="successful-access">
<section id="section-2.3.1.1">
            <h5 id="name-successful-access">
<a href="#section-2.3.1.1" class="section-number selfRef">2.3.1.1. </a><a href="#name-successful-access" class="section-name selfRef">Successful Access</a>
            </h5>
<p id="section-2.3.1.1-1"> If the crawler successfully downloads the robots.txt file, the
              crawler <span class="bcp14">MUST</span> follow the parseable rules.<a href="#section-2.3.1.1-1" class="pilcrow">¶</a></p>
</section>
</div>
<div id="redirects">
<section id="section-2.3.1.2">
            <h5 id="name-redirects">
<a href="#section-2.3.1.2" class="section-number selfRef">2.3.1.2. </a><a href="#name-redirects" class="section-name selfRef">Redirects</a>
            </h5>
<p id="section-2.3.1.2-1"> It's possible that a server responds to a robots.txt fetch
              request with a redirect, such as HTTP 301 or HTTP 302 in the
              case of HTTP. The crawlers <span class="bcp14">SHOULD</span> follow at
              least five consecutive redirects, even across authorities
              (for example, hosts in the case of HTTP).<a href="#section-2.3.1.2-1" class="pilcrow">¶</a></p>
<p id="section-2.3.1.2-2"> If a robots.txt file is reached within five consecutive
              redirects, the robots.txt file <span class="bcp14">MUST</span> be fetched,
              parsed, and its rules followed in the context of the initial
              authority.<a href="#section-2.3.1.2-2" class="pilcrow">¶</a></p>
<p id="section-2.3.1.2-3"> If there are more than five consecutive redirects, crawlers
              <span class="bcp14">MAY</span> assume that the robots.txt file is
              unavailable.<a href="#section-2.3.1.2-3" class="pilcrow">¶</a></p>
</section>
</div>
<div id="unavailable-status">
<section id="section-2.3.1.3">
            <h5 id="name-unavailable-status">
<a href="#section-2.3.1.3" class="section-number selfRef">2.3.1.3. </a><a href="#name-unavailable-status" class="section-name selfRef">"Unavailable" Status</a>
            </h5>
<p id="section-2.3.1.3-1"> "Unavailable" means the crawler tries to fetch the robots.txt file
              and the server responds with status codes indicating that the resource in question is unavailable. For
              example, in the context of HTTP, such status codes are
              in the 400-499 range.<a href="#section-2.3.1.3-1" class="pilcrow">¶</a></p>
<p id="section-2.3.1.3-2"> If a server status code indicates that the robots.txt file is
              unavailable to the crawler, then the crawler <span class="bcp14">MAY</span> access any
              resources on the server.<a href="#section-2.3.1.3-2" class="pilcrow">¶</a></p>
</section>
</div>
<div id="unreachable-status">
<section id="section-2.3.1.4">
            <h5 id="name-unreachable-status">
<a href="#section-2.3.1.4" class="section-number selfRef">2.3.1.4. </a><a href="#name-unreachable-status" class="section-name selfRef">"Unreachable" Status</a>
            </h5>
<p id="section-2.3.1.4-1"> If the robots.txt file is unreachable due to server or network
              errors, this means the robots.txt file is undefined and the crawler
              <span class="bcp14">MUST</span> assume complete disallow. For example, in
              the context of HTTP, server errors are identified by status codes
              in the 500-599 range.<a href="#section-2.3.1.4-1" class="pilcrow">¶</a></p>
<p id="section-2.3.1.4-2"> If the robots.txt file is undefined for a reasonably long period of
              time (for example, 30 days), crawlers <span class="bcp14">MAY</span> assume that
              the robots.txt file is unavailable as defined in
              <a href="#unavailable-status" class="xref">Section 2.3.1.3</a> or continue to use a cached
              copy.<a href="#section-2.3.1.4-2" class="pilcrow">¶</a></p>
</section>
</div>
<div id="parsing-errors">
<section id="section-2.3.1.5">
            <h5 id="name-parsing-errors">
<a href="#section-2.3.1.5" class="section-number selfRef">2.3.1.5. </a><a href="#name-parsing-errors" class="section-name selfRef">Parsing Errors</a>
            </h5>
<p id="section-2.3.1.5-1"> Crawlers <span class="bcp14">MUST</span> try to parse each line of the
              robots.txt file. Crawlers <span class="bcp14">MUST</span> use the parseable
              rules.<a href="#section-2.3.1.5-1" class="pilcrow">¶</a></p>
</section>
</div>
</section>
</div>
</section>
</div>
<div id="caching">
<section id="section-2.4">
        <h3 id="name-caching">
<a href="#section-2.4" class="section-number selfRef">2.4. </a><a href="#name-caching" class="section-name selfRef">Caching</a>
        </h3>
<p id="section-2.4-1"> Crawlers <span class="bcp14">MAY</span> cache the fetched robots.txt file's
          contents. Crawlers <span class="bcp14">MAY</span> use standard cache control as
          defined in <span>[<a href="#RFC9111" class="xref">RFC9111</a>]</span>. Crawlers
          <span class="bcp14">SHOULD NOT</span> use the cached version for more than 24
          hours, unless the robots.txt file is unreachable.<a href="#section-2.4-1" class="pilcrow">¶</a></p>
</section>
</div>
<div id="limits">
<section id="section-2.5">
        <h3 id="name-limits">
<a href="#section-2.5" class="section-number selfRef">2.5. </a><a href="#name-limits" class="section-name selfRef">Limits</a>
        </h3>
<p id="section-2.5-1"> Crawlers <span class="bcp14">SHOULD</span> impose a parsing limit to protect their systems;
          see <a href="#security" class="xref">Section 3</a>. The parsing limit <span class="bcp14">MUST</span> be at least
          500 kibibytes <span>[<a href="#KiB" class="xref">KiB</a>]</span>.<a href="#section-2.5-1" class="pilcrow">¶</a></p>
</section>
</div>
</section>
</div>
<div id="security">
<section id="section-3">
      <h2 id="name-security-considerations">
<a href="#section-3" class="section-number selfRef">3. </a><a href="#name-security-considerations" class="section-name selfRef">Security Considerations</a>
      </h2>
<p id="section-3-1"> The Robots Exclusion Protocol is not a substitute for valid
          content security measures. Listing paths in the robots.txt file
          exposes them publicly and thus makes the paths discoverable. To
          control access to the URI paths in a robots.txt file, users of
          the protocol should employ a valid security measure relevant to
          the application layer on which the robots.txt file is served --
          for example, in the case of HTTP, HTTP Authentication as defined in
          <span>[<a href="#RFC9110" class="xref">RFC9110</a>]</span>.<a href="#section-3-1" class="pilcrow">¶</a></p>
<p id="section-3-2"> To protect against attacks against their system, implementors
          of robots.txt parsing and matching logic should take the
          following considerations into account:<a href="#section-3-2" class="pilcrow">¶</a></p>
<span class="break"></span><dl class="dlParallel" id="section-3-3">
        <dt id="section-3-3.1"> Memory management:</dt>
        <dd style="margin-left: 1.5em" id="section-3-3.2">
          <a href="#limits" class="xref">Section 2.5</a> defines the lower
              limit of bytes that must be processed, which inherently also
              protects the parser from out-of-memory scenarios.<a href="#section-3-3.2" class="pilcrow">¶</a>
</dd>
        <dd class="break"></dd>
<dt id="section-3-3.3"> Invalid characters:</dt>
        <dd style="margin-left: 1.5em" id="section-3-3.4">
          <a href="#formal-syntax" class="xref">Section 2.2</a> defines
              a set of characters that parsers and matchers can expect in
              robots.txt files. Out-of-bound characters should be rejected
              as invalid, which limits the available attack vectors that
              attempt to compromise the system.<a href="#section-3-3.4" class="pilcrow">¶</a>
</dd>
        <dd class="break"></dd>
<dt id="section-3-3.5"> Untrusted content:</dt>
        <dd style="margin-left: 1.5em" id="section-3-3.6"> Implementors should treat the content of
              a robots.txt file as untrusted content, as defined by the
              specification of the application layer used. For example,
              in the context of HTTP, implementors should follow the
              Security Considerations section of
              <span>[<a href="#RFC9110" class="xref">RFC9110</a>]</span>.<a href="#section-3-3.6" class="pilcrow">¶</a>
</dd>
      <dd class="break"></dd>
</dl>
</section>
</div>
<div id="IANA">
<section id="section-4">
      <h2 id="name-iana-considerations">
<a href="#section-4" class="section-number selfRef">4. </a><a href="#name-iana-considerations" class="section-name selfRef">IANA Considerations</a>
      </h2>
<p id="section-4-1"> This document has no IANA actions.<a href="#section-4-1" class="pilcrow">¶</a></p>
</section>
</div>
<div id="examples">
<section id="section-5">
      <h2 id="name-examples">
<a href="#section-5" class="section-number selfRef">5. </a><a href="#name-examples" class="section-name selfRef">Examples</a>
      </h2>
<div id="simple-example">
<section id="section-5.1">
        <h3 id="name-simple-example">
<a href="#section-5.1" class="section-number selfRef">5.1. </a><a href="#name-simple-example" class="section-name selfRef">Simple Example</a>
        </h3>
<p id="section-5.1-1"> The following example shows:<a href="#section-5.1-1" class="pilcrow">¶</a></p>
<span class="break"></span><dl class="dlParallel" id="section-5.1-2">
          <dt id="section-5.1-2.1"> *:</dt>
          <dd style="margin-left: 1.5em" id="section-5.1-2.2"> A group that's relevant to all user agents that
                don't have an explicitly defined matching group. It allows
                access to the URLs with the /publications/ path prefix, and it
                restricts access to the URLs with the /example/ path prefix
                and to all URLs with a .gif suffix. The "*" character designates
                any character, including the otherwise-required forward
                slash; see <a href="#formal-syntax" class="xref">Section 2.2</a>.<a href="#section-5.1-2.2" class="pilcrow">¶</a>
</dd>
          <dd class="break"></dd>
<dt id="section-5.1-2.3"> foobot:</dt>
          <dd style="margin-left: 1.5em" id="section-5.1-2.4"> A regular case. A single user agent followed
                by rules. The crawler only has access to two URL path
                prefixes on the site -- /example/page.html and
                /example/allowed.gif. The rules of the group are missing
                the optional space character, which is acceptable as
                defined in <a href="#formal-syntax" class="xref">Section 2.2</a>.<a href="#section-5.1-2.4" class="pilcrow">¶</a>
</dd>
          <dd class="break"></dd>
<dt id="section-5.1-2.5"> barbot and bazbot:</dt>
          <dd style="margin-left: 1.5em" id="section-5.1-2.6"> A group that's relevant for more
                than one user agent. The crawlers are not allowed to access
                the URLs with the /example/page.html path prefix but
                otherwise have unrestricted access to the rest of the URLs
                on the site.<a href="#section-5.1-2.6" class="pilcrow">¶</a>
</dd>
          <dd class="break"></dd>
<dt id="section-5.1-2.7"> quxbot:</dt>
          <dd style="margin-left: 1.5em" id="section-5.1-2.8"> An empty group at the end of the file. The crawler has
                unrestricted access to the URLs on the site.<a href="#section-5.1-2.8" class="pilcrow">¶</a>
</dd>
        <dd class="break"></dd>
</dl>
<div class="alignLeft art-text artwork" id="section-5.1-3">
<pre>
            User-Agent: *
            Disallow: *.gif$
            Disallow: /example/
            Allow: /publications/

            User-Agent: foobot
            Disallow:/
            Allow:/example/page.html
            Allow:/example/allowed.gif

            User-Agent: barbot
            User-Agent: bazbot
            Disallow: /example/page.html

            User-Agent: quxbot

            EOF
</pre><a href="#section-5.1-3" class="pilcrow">¶</a>
</div>
</section>
</div>
<div id="longest-match">
<section id="section-5.2">
        <h3 id="name-longest-match">
<a href="#section-5.2" class="section-number selfRef">5.2. </a><a href="#name-longest-match" class="section-name selfRef">Longest Match</a>
        </h3>
<p id="section-5.2-1"> The following example shows that in the case of two rules, the
            longest one is used for matching. In the following case,
            /example/page/disallowed.gif <span class="bcp14">MUST</span> be used for
            the URI example.com/example/page/disallow.gif.<a href="#section-5.2-1" class="pilcrow">¶</a></p>
<div class="alignLeft art-text artwork" id="section-5.2-2">
<pre>
            User-Agent: foobot
            Allow: /example/page/
            Disallow: /example/page/disallowed.gif
</pre><a href="#section-5.2-2" class="pilcrow">¶</a>
</div>
</section>
</div>
</section>
</div>
<section id="section-6">
      <h2 id="name-references">
<a href="#section-6" class="section-number selfRef">6. </a><a href="#name-references" class="section-name selfRef">References</a>
      </h2>
<section id="section-6.1">
        <h3 id="name-normative-references">
<a href="#section-6.1" class="section-number selfRef">6.1. </a><a href="#name-normative-references" class="section-name selfRef">Normative References</a>
        </h3>
<dl class="references">
<dt id="RFC2046">[RFC2046]</dt>
        <dd>
<span class="refAuthor">Freed, N.</span> and <span class="refAuthor">N. Borenstein</span>, <span class="refTitle">"Multipurpose Internet Mail Extensions (MIME) Part Two: Media Types"</span>, <span class="seriesInfo">RFC 2046</span>, <span class="seriesInfo">DOI 10.17487/RFC2046</span>, <time datetime="1996-11" class="refDate">November 1996</time>, <span>&lt;<a href="https://www.rfc-editor.org/info/rfc2046">https://www.rfc-editor.org/info/rfc2046</a>&gt;</span>. </dd>
<dd class="break"></dd>
<dt id="RFC2119">[RFC2119]</dt>
        <dd>
<span class="refAuthor">Bradner, S.</span>, <span class="refTitle">"Key words for use in RFCs to Indicate Requirement Levels"</span>, <span class="seriesInfo">BCP 14</span>, <span class="seriesInfo">RFC 2119</span>, <span class="seriesInfo">DOI 10.17487/RFC2119</span>, <time datetime="1997-03" class="refDate">March 1997</time>, <span>&lt;<a href="https://www.rfc-editor.org/info/rfc2119">https://www.rfc-editor.org/info/rfc2119</a>&gt;</span>. </dd>
<dd class="break"></dd>
<dt id="RFC3629">[RFC3629]</dt>
        <dd>
<span class="refAuthor">Yergeau, F.</span>, <span class="refTitle">"UTF-8, a transformation format of ISO 10646"</span>, <span class="seriesInfo">STD 63</span>, <span class="seriesInfo">RFC 3629</span>, <span class="seriesInfo">DOI 10.17487/RFC3629</span>, <time datetime="2003-11" class="refDate">November 2003</time>, <span>&lt;<a href="https://www.rfc-editor.org/info/rfc3629">https://www.rfc-editor.org/info/rfc3629</a>&gt;</span>. </dd>
<dd class="break"></dd>
<dt id="RFC3986">[RFC3986]</dt>
        <dd>
<span class="refAuthor">Berners-Lee, T.</span>, <span class="refAuthor">Fielding, R.</span>, and <span class="refAuthor">L. Masinter</span>, <span class="refTitle">"Uniform Resource Identifier (URI): Generic Syntax"</span>, <span class="seriesInfo">STD 66</span>, <span class="seriesInfo">RFC 3986</span>, <span class="seriesInfo">DOI 10.17487/RFC3986</span>, <time datetime="2005-01" class="refDate">January 2005</time>, <span>&lt;<a href="https://www.rfc-editor.org/info/rfc3986">https://www.rfc-editor.org/info/rfc3986</a>&gt;</span>. </dd>
<dd class="break"></dd>
<dt id="RFC5234">[RFC5234]</dt>
        <dd>
<span class="refAuthor">Crocker, D., Ed.</span> and <span class="refAuthor">P. Overell</span>, <span class="refTitle">"Augmented BNF for Syntax Specifications: ABNF"</span>, <span class="seriesInfo">STD 68</span>, <span class="seriesInfo">RFC 5234</span>, <span class="seriesInfo">DOI 10.17487/RFC5234</span>, <time datetime="2008-01" class="refDate">January 2008</time>, <span>&lt;<a href="https://www.rfc-editor.org/info/rfc5234">https://www.rfc-editor.org/info/rfc5234</a>&gt;</span>. </dd>
<dd class="break"></dd>
<dt id="RFC8174">[RFC8174]</dt>
        <dd>
<span class="refAuthor">Leiba, B.</span>, <span class="refTitle">"Ambiguity of Uppercase vs Lowercase in RFC 2119 Key Words"</span>, <span class="seriesInfo">BCP 14</span>, <span class="seriesInfo">RFC 8174</span>, <span class="seriesInfo">DOI 10.17487/RFC8174</span>, <time datetime="2017-05" class="refDate">May 2017</time>, <span>&lt;<a href="https://www.rfc-editor.org/info/rfc8174">https://www.rfc-editor.org/info/rfc8174</a>&gt;</span>. </dd>
<dd class="break"></dd>
<dt id="RFC8288">[RFC8288]</dt>
        <dd>
<span class="refAuthor">Nottingham, M.</span>, <span class="refTitle">"Web Linking"</span>, <span class="seriesInfo">RFC 8288</span>, <span class="seriesInfo">DOI 10.17487/RFC8288</span>, <time datetime="2017-10" class="refDate">October 2017</time>, <span>&lt;<a href="https://www.rfc-editor.org/info/rfc8288">https://www.rfc-editor.org/info/rfc8288</a>&gt;</span>. </dd>
<dd class="break"></dd>
<dt id="RFC9110">[RFC9110]</dt>
        <dd>
<span class="refAuthor">Fielding, R., Ed.</span>, <span class="refAuthor">Nottingham, M., Ed.</span>, and <span class="refAuthor">J. Reschke, Ed.</span>, <span class="refTitle">"HTTP Semantics"</span>, <span class="seriesInfo">STD 97</span>, <span class="seriesInfo">RFC 9110</span>, <span class="seriesInfo">DOI 10.17487/RFC9110</span>, <time datetime="2022-06" class="refDate">June 2022</time>, <span>&lt;<a href="https://www.rfc-editor.org/info/rfc9110">https://www.rfc-editor.org/info/rfc9110</a>&gt;</span>. </dd>
<dd class="break"></dd>
<dt id="RFC9111">[RFC9111]</dt>
      <dd>
<span class="refAuthor">Fielding, R., Ed.</span>, <span class="refAuthor">Nottingham, M., Ed.</span>, and <span class="refAuthor">J. Reschke, Ed.</span>, <span class="refTitle">"HTTP Caching"</span>, <span class="seriesInfo">STD 98</span>, <span class="seriesInfo">RFC 9111</span>, <span class="seriesInfo">DOI 10.17487/RFC9111</span>, <time datetime="2022-06" class="refDate">June 2022</time>, <span>&lt;<a href="https://www.rfc-editor.org/info/rfc9111">https://www.rfc-editor.org/info/rfc9111</a>&gt;</span>. </dd>
<dd class="break"></dd>
</dl>
</section>
<section id="section-6.2">
        <h3 id="name-informative-references">
<a href="#section-6.2" class="section-number selfRef">6.2. </a><a href="#name-informative-references" class="section-name selfRef">Informative References</a>
        </h3>
<dl class="references">
<dt id="KiB">[KiB]</dt>
        <dd>
<span class="refTitle">"Kibibyte"</span>, <span class="refContent">Simple English Wikipedia, the free encyclopedia</span>, <time datetime="2020-09-17" class="refDate">17 September 2020</time>, <span>&lt;<a href="https://simple.wikipedia.org/wiki/Kibibyte">https://simple.wikipedia.org/wiki/Kibibyte</a>&gt;</span>. </dd>
<dd class="break"></dd>
<dt id="ROBOTSTXT">[ROBOTSTXT]</dt>
        <dd>
<span class="refTitle">"The Web Robots Pages (including /robots.txt)"</span>, <span>2007</span>, <span>&lt;<a href="https://www.robotstxt.org/">https://www.robotstxt.org/</a>&gt;</span>. </dd>
<dd class="break"></dd>
<dt id="SITEMAPS">[SITEMAPS]</dt>
      <dd>
<span class="refTitle">"What are Sitemaps? (Sitemap protocol)"</span>, <span>April 2020</span>, <span>&lt;<a href="https://www.sitemaps.org/index.html">https://www.sitemaps.org/index.html</a>&gt;</span>. </dd>
<dd class="break"></dd>
</dl>
</section>
</section>
<div id="authors-addresses">
<section id="appendix-A">
      <h2 id="name-authors-addresses">
<a href="#name-authors-addresses" class="section-name selfRef">Authors' Addresses</a>
      </h2>
<address class="vcard">
        <div dir="auto" class="left"><span class="fn nameRole">Martijn Koster</span></div>
<div dir="auto" class="left"><span class="extended-address">Stalworthy Manor Farm</span></div>
<div dir="auto" class="left"><span class="street-address">Suton Lane</span></div>
<div dir="auto" class="left"><span class="locality">Wymondham, Norfolk</span></div>
<div dir="auto" class="left"><span class="postal-code">NR18 9JG</span></div>
<div dir="auto" class="left"><span class="country-name">United Kingdom</span></div>
<div class="email">
<span>Email:</span>
<a href="mailto:m.koster@greenhills.co.uk" class="email">m.koster@greenhills.co.uk</a>
</div>
</address>
<address class="vcard">
        <div dir="auto" class="left"><span class="fn nameRole">Gary Illyes</span></div>
<div dir="auto" class="left"><span class="org">Google LLC</span></div>
<div dir="auto" class="left"><span class="street-address">Brandschenkestrasse 110</span></div>
<div dir="auto" class="left">CH-<span class="postal-code">8002</span> <span class="locality">Zürich</span>
</div>
<div dir="auto" class="left"><span class="country-name">Switzerland</span></div>
<div class="email">
<span>Email:</span>
<a href="mailto:garyillyes@google.com" class="email">garyillyes@google.com</a>
</div>
</address>
<address class="vcard">
        <div dir="auto" class="left"><span class="fn nameRole">Henner Zeller</span></div>
<div dir="auto" class="left"><span class="org">Google LLC</span></div>
<div dir="auto" class="left"><span class="street-address">1600 Amphitheatre Pkwy</span></div>
<div dir="auto" class="left">
<span class="locality">Mountain View</span>, <span class="region">CA</span> <span class="postal-code">94043</span>
</div>
<div dir="auto" class="left"><span class="country-name">United States of America</span></div>
<div class="email">
<span>Email:</span>
<a href="mailto:henner@google.com" class="email">henner@google.com</a>
</div>
</address>
<address class="vcard">
        <div dir="auto" class="left"><span class="fn nameRole">Lizzi Sassman</span></div>
<div dir="auto" class="left"><span class="org">Google LLC</span></div>
<div dir="auto" class="left"><span class="street-address">Brandschenkestrasse 110</span></div>
<div dir="auto" class="left">CH-<span class="postal-code">8002</span> <span class="locality">Zürich</span>
</div>
<div dir="auto" class="left"><span class="country-name">Switzerland</span></div>
<div class="email">
<span>Email:</span>
<a href="mailto:lizzi@google.com" class="email">lizzi@google.com</a>
</div>
</address>
</section>
</div>
<script>const toc = document.getElementById("toc");
toc.querySelector("h2").addEventListener("click", e => {
  toc.classList.toggle("active");
});
toc.querySelector("nav").addEventListener("click", e => {
  toc.classList.remove("active");
});
</script>
</body>
</html>