1d6a6f0aeff1bb8556c6f9c113639bff *DESCRIPTION
68f27e7ba14a298e399e771c8db22664 *LICENSE
d6631a9c6caf2369fd72772d6ae490ae *NAMESPACE
14dc967e76cbd22a976b1b56cad79cf6 *NEWS.md
d5436daf68a3991a73f8643b05fd5b60 *R/as_list.R
5752131497444a8e5f29b041ce1ee5a4 *R/fix_url.R
010daf52953a9ff2cbd5e61f51e9ec0e *R/get_robotstxt.R
fa69bc472c1bf995b0ceced3693ea988 *R/get_robotstxt_http_get.R
266cb3ca8e7efbd7d681f515f511b4b0 *R/get_robotstxts.R
7472469d031d4cc751f68cccf5dcc424 *R/guess_domain.R
b03e4c44889405d275625279bfc12fd5 *R/http_domain_changed.R
bc0518b97b3e619e16a05e2eeb894f54 *R/http_subdomain_changed.R
e8a80d92205f7a7c071fc2c33d3164d3 *R/http_was_redirected.R
7468fb212b249642092d23f738d4e4e4 *R/is_suspect_robotstxt.R
a03a6191f6358e3951f95618f09559f1 *R/is_valid_robotstxt.R
ee18a426bb4ab5372904af67cf68b1ad *R/list_merge.R
e3a0302720bae3778cbccd47abf8f421 *R/null_to_default.R
ad57b0dfc971640f22f80d92abd65ab1 *R/parse_robotstxt.R
d05dd321311fed569993641c37a87a4c *R/parse_url.R
4827a785499923cc2a1d1436e98488ac *R/paths_allowed.R
18e347fadbeaea7ebac55e371822ab98 *R/paths_allowed_worker_spiderbar.R
e002670688bb09fd0d464f674968fc65 *R/pipe.R
3df4af40a6b8b0073c2c913c2aea1afd *R/print_robotstxt.R
19f55e6e481adce4964967f46ed90f53 *R/print_robotstxt_text.R
1dd2e720f2caba61d07f8605315a6149 *R/remove_domain.R
4f22a36dbb066a315561b6c410940947 *R/request_handler_handler.R
8715b08d41048f33a6fbdfd0c0103f9d *R/robotstxt.R
7d34b0e8417b244318a5e39c5ae010ec *R/rt_cache.R
716845e77f07f6f8f91132581ab79845 *R/rt_get_comments.R
31c3879c99a5660fe3d93b30226643ee *R/rt_get_fields.R
ec72b13ccc25816f202e1fd05a25a4fc *R/rt_get_fields_worker.R
40d44e8efb0e17bc3daebdc2bb93dd2c *R/rt_get_useragent.R
7dd77adaa8a4b83277a60785a70793b8 *R/rt_request_handler.R
d6c7aa985fad525cece3e959964ed6a6 *R/rt_request_handler_defaults.R
9c483bb690f14cc30818c7509f324750 *R/sanitize_path.R
b1f74ca0815a34b1ef3718e05e2ab9ea *R/tools.R
bc92288733fad8da7cf63c992db859df *README.md
0595907652fce4870a485d37d9a57bad *build/vignette.rds
b40fbd096716a3f173644053c37e8438 *inst/doc/using_robotstxt.R
dc3e2e30f4cfd3d18bb19bfce9f61a6a *inst/doc/using_robotstxt.Rmd
c2a72d4c25cc90c507541118024593e6 *inst/doc/using_robotstxt.html
4600b105324e0e285ad6384f5f01968d *inst/http_requests/http_404.rds
48598696c69b2826c3e9d919c4377369 *inst/http_requests/http_client_error.rds
bb9f78439e4ded437aaac70a3967e79b *inst/http_requests/http_domain_change.rds
513a4bc54cbd540cfd4f1057a46d2d7e *inst/http_requests/http_html_content.rds
e8295be102459c31b63b9faee4e4c519 *inst/http_requests/http_ok_1.rds
cda4365dc8a9e29eb2623c9428be8047 *inst/http_requests/http_ok_2.rds
1dc61ada1a0a11297d653443e75b5b10 *inst/http_requests/http_ok_3.rds
44f05e9ab8d7dae5b744a5f60fcca5cc *inst/http_requests/http_ok_4.rds
09019d9676716e69aa98d8a8b02be6b0 *inst/http_requests/http_redirect_www.rds
b95430e784c92a5ef59af45e523ace00 *inst/http_requests/http_server_error.rds
4d8921c3dc225aa07cd9b3f66cde3aff *inst/robotstxts/allow_single_bot.txt
e2b27c9301b4f24c3b2db3f8886b2dc1 *inst/robotstxts/crawl_delay.txt
8cb7a8b11cc42f1f77aa0670c43071a3 *inst/robotstxts/disallow_all_for_BadBot.txt
d35cb09d5662e686347b00b959c1550f *inst/robotstxts/disallow_all_for_all.txt
542398cc4e150fd288cea549bdca16a4 *inst/robotstxts/disallow_some_for_all.txt
9ca8636f08a496a82ed0a64694d005c8 *inst/robotstxts/disallow_two_at_once.txt
7215ee9c7d9dc229d2921a40e899ec5f *inst/robotstxts/empty.txt
d034a6e564902694ac6b372940786d80 *inst/robotstxts/host.txt
95d8d7433b4cd37b5630cf68500127f0 *inst/robotstxts/robots_amazon.txt
a09430767142d30f771f9726196048e1 *inst/robotstxts/robots_bundestag.txt
7a58b13412a11e1c8650008ae15f2bef *inst/robotstxts/robots_cdc.txt
7a58b13412a11e1c8650008ae15f2bef *inst/robotstxts/robots_cdc2.txt
3b1a4b952779358ff7d6015b0dee0e68 *inst/robotstxts/robots_facebook.txt
c51eaab2c346b3a156c72097e52b10d9 *inst/robotstxts/robots_facebook_unsupported.txt
62dbaccb94d6ab8aa5e178339efbe127 *inst/robotstxts/robots_google.txt
658622dc6bda3645b94e37fa1ea38295 *inst/robotstxts/robots_new_york_times.txt
48a1701be03cf9590a8ef623c5f2e8e6 *inst/robotstxts/robots_pmeissner.txt
67dfdd8b9acd9363417c145388315dab *inst/robotstxts/robots_spiegel.txt
6ed07c3b696c3b4d47e1b3b4a42f5c7f *inst/robotstxts/robots_wikipedia.txt
b03c30fd553878fce698924ee1eca06e *inst/robotstxts/robots_wikipedia_20170706.txt
4637c3108489f0f0e5e283ef16884baf *inst/robotstxts/robots_yahoo.txt
6b8539213a6199601ee9d81c70a6ac31 *inst/robotstxts/selfhtml_Example.txt
2b9b53034510e3b66af8968145fe0a4f *inst/robotstxts/testing_comments.txt
a6f4f6d1bedffc16330a9d11020a0936 *inst/urls.txt
831093032e270a2f516380a89096b0a1 *man/as.list.robotstxt_text.Rd
e5a08bdc08a56d50e1993b7b56a8f0d0 *man/fix_url.Rd
be9ee5d0eb602d68600433d1f7f963b6 *man/get_robotstxt.Rd
7f888b3f325774cfb7359eb6a2160609 *man/get_robotstxt_http_get.Rd
6ab809fc41dc93147324e23fb3580a4d *man/get_robotstxts.Rd
ef76b13de0d135509f434f55a4e1e813 *man/guess_domain.Rd
b5c0a8e40d5be10f3e472ff2c21c4028 *man/http_domain_changed.Rd
cfded2d7606869affa85fec985320e86 *man/http_subdomain_changed.Rd
38f51200d9381e52bca70c994c72d690 *man/http_was_redirected.Rd
1c14deb3dfa2c975b2dafef9adf10ad2 *man/is_suspect_robotstxt.Rd
2e7a8f80761cec4ef8d554721b31752a *man/is_valid_robotstxt.Rd
e95b0adb4e98e040776320413d1d0cb6 *man/list_merge.Rd
55d3701de26b6f6f6f0acdaf52a71e3a *man/named_list.Rd
8abdc579ccd16a0886a9540436843dee *man/null_to_defeault.Rd
3d89b9818db895814ac6895b2ff3111a *man/parse_robotstxt.Rd
966edd9838c9552d49b0de74a92472ba *man/parse_url.Rd
6ca84d3cd4bb28e805d926642d603d1a *man/paths_allowed.Rd
ed8a624cfd48e11ae47815b252bd0c6e *man/paths_allowed_worker_spiderbar.Rd
8b5c95ad652485cdb6d3c0bd99d99396 *man/pipe.Rd
5d4ddc9669caa3cc284e17a0876f8320 *man/print.robotstxt.Rd
eca8dd7a192d83c834c7bba7caced8b8 *man/print.robotstxt_text.Rd
08af33a748555ccb6911671bb646a9f9 *man/remove_domain.Rd
3946561a85a72c72f37c68633dceef65 *man/request_handler_handler.Rd
f12b9529edfe7a1e431f4fc897842427 *man/robotstxt.Rd
936f88d83b62e768a0924536e3f91173 *man/rt_cache.Rd
11a1c6f616b272b2c96357e07120d5b4 *man/rt_get_comments.Rd
e2d090c1bf4c35e681da0f714e450e0e *man/rt_get_fields.Rd
2f946d9ef928f260d6f5c3600476d7b8 *man/rt_get_fields_worker.Rd
3a762d0e69b3478eaa7ec738345c91ba *man/rt_get_rtxt.Rd
61a327662e2ed8433de9908d5a6d2969 *man/rt_get_useragent.Rd
b8523fd7dca3c50be9e742f7ab8abfcf *man/rt_list_rtxt.Rd
18b4bc0d8fb701155d8c688fdeb0c01d *man/rt_request_handler.Rd
686a909015071ec483e0199f81043006 *man/sanitize_path.Rd
480e44e1c814faf1886f057b8a375590 *tests/testthat.R
665c13af09e4842fa3dea96e9af46430 *tests/testthat/test_attribute_handling.R
de71a1bf36ceb10d1bf25fac5cfe5580 *tests/testthat/test_get_robotstxt.R
b8a10ccb7d45faf65296b25e0d6a7aff *tests/testthat/test_http_event_handling.R
a3ae813bf396e4e35709fac4ba1ac9fc *tests/testthat/test_issue50.R
8ae85233dc20826cc031b030ac47d512 *tests/testthat/test_parser.R
42a5375d4cfde4e236ad31945fa51869 *tests/testthat/test_paths_allowed.R
e600f6540b6eb7b9b95074ddfb5a3157 *tests/testthat/test_robotstxt.R
4cfa525d518ef93fcfa0edea85341f78 *tests/testthat/test_tools.R
f8b3499a876f34f4164a0cc4fb47092f *vignettes/style.css
dc3e2e30f4cfd3d18bb19bfce9f61a6a *vignettes/using_robotstxt.Rmd
