diff --git a/script/derekovecs-server b/script/derekovecs-server
index d460345..bc66b01 100755
--- a/script/derekovecs-server
+++ b/script/derekovecs-server
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 
-our $VERSION = '0.96';
+our $VERSION = '0.97';
 
 use IDS::DeReKoVecs::Read;
 use Mojolicious::Lite;
@@ -61,6 +61,78 @@
 our $opt_p = 5676;
 our $opt_C;
 
+# Crawlers that collect training data for language models walk the
+# parameterised URLs of this server, and those are the most expensive thing it
+# does: every word is a URL of its own, every result links to the next words,
+# so the crawl space is the whole vocabulary and nothing of it can be served
+# from a cache. They are turned away before anything is computed.
+#
+# The list is a regular expression and can be replaced in the configuration
+# file, robots => {block_user_agents => '...'}; an empty value switches the
+# blocking off. Ordinary search engines are deliberately not in it - they are
+# kept out of the parameterised URLs by robots.txt and X-Robots-Tag instead,
+# and the entry page stays indexable.
+my $DEFAULT_BLOCKED_AGENTS = join('|',
+  'AI2Bot', 'Amazonbot', 'anthropic-ai', 'Bytespider', 'CCBot', 'ChatGPT-User',
+  'Claude-SearchBot', 'Claude-User', 'Claude-Web', 'ClaudeBot', 'cohere-ai',
+  'cohere-training-data-crawler', 'Diffbot', 'DuckAssistBot', 'FacebookBot',
+  'facebookexternalhit', 'FriendlyCrawler', 'GPTBot', 'ImagesiftBot',
+  'Kangaroo Bot', 'Meta-ExternalAgent', 'Meta-ExternalFetcher', 'meta-webindexer',
+  'OAI-SearchBot', 'omgili', 'PanguBot', 'Perplexity-User', 'PerplexityBot',
+  'Scrapy', 'SemrushBot-OCOB', 'Timpibot', 'TikTokSpider', 'YouBot'
+);
+
+my $blocked_agents = app->config->{robots}->{block_user_agents} // $DEFAULT_BLOCKED_AGENTS;
+my $blocked_agents_re = length($blocked_agents) ? qr/(?i:$blocked_agents)/ : undef;
+
+# robots.txt for the ones that read it. A reverse proxy that mounts derekovecs
+# below a path has to serve the site wide /robots.txt itself, crawlers only
+# ask for it at the root of the host - this one is for installations that own
+# their host name, and does no harm anywhere else.
+my $ROBOTS_TXT = join("\n",
+  '# Every URL with parameters is a result computed for one word, and the',
+  '# vocabulary is large enough that following them is an endless crawl.',
+  'User-agent: *',
+  'Disallow: /*?',
+  'Crawl-delay: 10',
+  '',
+  '# Collecting training data for language models is not what this service is',
+  '# here for. Please use the corpus itself, https://www.ids-mannheim.de/dereko',
+  (map { "User-agent: $_" }
+     (split(/\|/, $DEFAULT_BLOCKED_AGENTS),
+      # tokens that are only understood in robots.txt, there is no crawler of
+      # that name to match against
+      'Google-Extended', 'Applebot-Extended')),
+  'Disallow: /',
+  ''
+);
+
+hook before_dispatch => sub {
+  my $c = shift;
+
+  if ($c->req->url->path->to_string =~ m@(?:^|/)robots\.txt$@) {
+    return $c->render(data => $ROBOTS_TXT, format => 'txt');
+  }
+
+  # Only the entry page is worth indexing, everything else is computed.
+  if (length($c->req->url->query->to_string)) {
+    $c->res->headers->header('X-Robots-Tag' => 'noindex, nofollow');
+    $c->stash(noindex => 1);
+  }
+
+  my $ua = $c->req->headers->user_agent;
+  if ($blocked_agents_re && defined($ua) && $ua =~ $blocked_agents_re) {
+    $c->app->log->info('Blocked crawler ' . $c->remote_addr . ' "' . $ua . '" ' . $c->req->url);
+    $c->res->headers->header('X-Robots-Tag' => 'noindex, nofollow');
+    return $c->render(
+      text   => "This service computes an answer for every request and is not a source of training data.\n"
+              . "The corpus behind it is documented at https://www.ids-mannheim.de/dereko\n",
+      status => 403,
+      format => 'txt'
+    );
+  }
+};
+
 # Neighbourhood results are the biggest objects the server keeps around (one
 # hash plus one vector array per neighbour), so this cache is deliberately
 # small. It is per worker, i.e. the total footprint is