)]}'
{
  "commit": "b3cd59f0a422e7fcdf78658d4480966f61800bd1",
  "tree": "2a02147bf42a94cd2c6b67c1a83ae209a3f41c25",
  "parents": [
    "23993b6d7a6cfcc85238bc6ba3170cd55fa88289"
  ],
  "author": {
    "name": "Marc Kupietz",
    "email": "kupietz@ids-mannheim.de",
    "time": "Fri Aug 21 10:47:45 2026 +0200"
  },
  "committer": {
    "name": "Marc Kupietz",
    "email": "kupietz@ids-mannheim.de",
    "time": "Fri Aug 21 10:47:45 2026 +0200"
  },
  "message": "Turn away the crawlers that collect training data\n\nEvery URL with parameters is a result computed for one word, there is one\nfor every word of the vocabulary, and none of it can be answered from a\ncache, so a crawler that follows them keeps the machine busy for nothing.\nThe instance on corpora.ids-mannheim.de answers about 21000 such requests\na day.\n\nRequests whose User-Agent matches a list of such crawlers now get 403 in\nbefore_dispatch, i.e. before any computation. The list can be replaced in\nthe configuration file, robots \u003d\u003e {block_user_agents \u003d\u003e \u0027...\u0027}, and an\nempty value switches the blocking off. Ordinary search engines are not on\nit.\n\nFor the crawlers that do read robots.txt, every response to a request\nwith parameters carries X-Robots-Tag: noindex, nofollow, result pages\ncarry the matching meta element, and robots.txt is served at every path -\nwhich only helps where derekovecs owns its host name, a proxy that mounts\nit below a path still has to serve the site wide one itself.\n\nCo-Authored-By: Claude Opus 5 \u003cnoreply@anthropic.com\u003e\nChange-Id: Id68487ef716459e0ea9f045060edc34edf85065e\n",
  "tree_diff": [
    {
      "type": "modify",
      "old_id": "f72fb1336e01f327b809443f5dab39993d761b70",
      "old_mode": 33188,
      "old_path": "Changelog.md",
      "new_id": "652474b965c6ccf7d63e865e3a4dfde657613832",
      "new_mode": 33188,
      "new_path": "Changelog.md"
    },
    {
      "type": "modify",
      "old_id": "b6e0af3c35b64127162c1390dc0405c29a808c50",
      "old_mode": 33188,
      "old_path": "README.md",
      "new_id": "346c4ea2c6c55fc47d9c33a255fdb471b28f7b17",
      "new_mode": 33188,
      "new_path": "README.md"
    },
    {
      "type": "modify",
      "old_id": "34d62977e108036ca9f59c71b8ae4a14be4a3fab",
      "old_mode": 33188,
      "old_path": "example.conf",
      "new_id": "e999b3dc75b81151cda6078304033a02f629346f",
      "new_mode": 33188,
      "new_path": "example.conf"
    },
    {
      "type": "modify",
      "old_id": "d46034542c7cdd7f9b03ddbbb95c08c560529960",
      "old_mode": 33261,
      "old_path": "script/derekovecs-server",
      "new_id": "bc66b0129c856d30f56df654c2a41dafa75d4dab",
      "new_mode": 33261,
      "new_path": "script/derekovecs-server"
    },
    {
      "type": "modify",
      "old_id": "873c0a78e27e105dde560783d283c464f91b4f9f",
      "old_mode": 33188,
      "old_path": "templates/index.html.ep",
      "new_id": "646ddf51422e769c26bfcd1a19c472eb578a52f3",
      "new_mode": 33188,
      "new_path": "templates/index.html.ep"
    }
  ]
}