Record Path        URL path         Matches
                                /tmp               /tmp               yes
                                /tmp               /tmp.html          yes
                                /tmp               /tmp/a.html        yes
                                /tmp/              /tmp               no
                                /tmp/              /tmp/              yes
                                /tmp/              /tmp/a.html        yes
     
                                /a%3cd.html        /a%3cd.html        yes
                                /a%3Cd.html        /a%3cd.html        yes
                                /a%3cd.html        /a%3Cd.html        yes
                                /a%3Cd.html        /a%3Cd.html        yes
     
                                /a%2fb.html        /a%2fb.html        yes
                                /a%2fb.html        /a/b.html          no
                                /a/b.html          /a%2fb.html        no
                                /a/b.html          /a/b.html          yes
     
                                /%7esubculture/index.html /~subculture/index.html   yes
                                /~subculture/index.html   /%7Esubculture/index.html yes
    
                                robotstxt    = *blankcomment
                                *blankcomment record *( 1*commentblank 1*record )
                                              *blankcomment
                                blankcomment = 1*(blank | commentline)
                                commentblank = *commentline blank *(blankcomment)
                                blank        = *space CRLF
                                CRLF         = CR LF
                                record       = *commentline agentline *(commentline | agentline)                                        
                                              1*ruleline *(commentline | ruleline)
                                agentline    = "User-agent:" *space agent  [comment] CRLF
                                ruleline     = (disallowline | allowline | extension)
                                disallowline = "Disallow" ":" *space path [comment] CRLF
                                allowline    = "Allow" ":" *space rpath [comment] CRLF
                                extension    = token : *space value [comment] CRLF
                                value        = <any CHAR except CR or LF or "#">