feat: add transfers crawler

README.md

@@ -79,6 +79,19 @@ echo '{"type":"competition","competition_type":"world_cup","href":"/world-cup/st
 # scrape games for UEFA Euro 2024 (season=2023 on Transfermarkt)
 echo '{"type":"competition","competition_type":"uefa_euro","href":"/uefa-euro/startseite/pokalwettbewerb/EURO","competition_name":"UEFA Euro"}' \
     | python -m tfmkt games --season 2023 > euro_2024_games.json
+
+# scrape transfers for a competition (paste the path from any Transfermarkt URL)
+echo '{"href":"/premier-league/transfers/wettbewerb/GB1/plus/?saison_id=2025&s_w=&leihe=1&intern=0&intern=1"}' \
+    | python -m tfmkt transfers > transfers.json
+
+# scrape transfers for a single club
+echo '{"href":"/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}' \
+    | python -m tfmkt transfers > transfers.json
+
+# chain: scrape clubs then their transfers
+cat competitions.json | head -1 \
+    | python -m tfmkt clubs \
+    | python -m tfmkt transfers > transfers.json
 ```
 
 Alternatively you can also use [`dcaribou/transfermarkt-scraper`](https://hub.docker.com/repository/docker/dcaribou/transfermarkt-scraper) docker image
@@ -106,6 +119,7 @@ Items are extracted in JSON format with one JSON object per item, which get prin
 | `tournament_editions` | Competition | Tournament Edition | Historical editions with year, season, winner, coach |
 | `games` | Competition | Game | Match result, events, managers. Use `--season` to select the edition (e.g. `--season 2021` for Qatar 2022, `--season 2023` for Euro 2024) |
 | `game_lineups` | Game | Game Lineups | Starting XI, substitutes, formation |
+| `transfers` | Club or Competition | Transfer | Arrivals and departures with fee, market value, and player href |
 
 Check out [transfermarkt-datasets](https://github.com/dcaribou/transfermarkt-datasets) to see `transfermarkt-scraper` in action on a real project.
 

samples/transfers.json

@@ -0,0 +1,3 @@
+{"type": "transfer", "club": "Cruzeiro Esporte Clube", "direction": "In", "player": "Gerson", "player_href": "/gerson/profil/spieler/341705", "age": "28", "nationality": "Brazil", "position": "Central Midfield", "market_value": "", "origin_club": "Zenit S-Pb", "fee": "\u20ac27.00m", "parent": {"href": "/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}, "source": "https://www.transfermarkt.co.uk/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}
+{"type": "transfer", "club": "Cruzeiro Esporte Clube", "direction": "In", "player": "Keny Arroyo", "player_href": "/keny-arroyo/profil/spieler/1074584", "age": "19", "nationality": "Ecuador", "position": "Right Winger", "market_value": "", "origin_club": "Besiktas", "fee": "\u20ac8.50m", "parent": {"href": "/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}, "source": "https://www.transfermarkt.co.uk/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}
+{"type": "transfer", "club": "Cruzeiro Esporte Clube", "direction": "In", "player": "Marquinhos", "player_href": "/marquinhos/profil/spieler/668268", "age": "22", "nationality": "Brazil", "position": "Right Winger", "market_value": "", "origin_club": "Arsenal", "fee": "\u20ac3.00m", "parent": {"href": "/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}, "source": "https://www.transfermarkt.co.uk/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}

tests/test_crawlers.py

@@ -468,6 +468,67 @@ def test_tournament_editions(tmp_path):
     assert item_2022["season"] == "2021"
     assert item_2022["winner"] == "Argentina"
 
+# ---------------------------------------------------------------------------
+# 12. Transfers (1 request — 1 small club)
+# ---------------------------------------------------------------------------
+
+def test_transfers(tmp_path):
+    """Feed a single small club (HNK Sibenik) to get its transfer activity."""
+    items = run_crawler(
+        "transfers",
+        parents_data={
+            "type": "club",
+            "href": "/hnk-sibenik/startseite/verein/223",
+        },
+        tmp_path=tmp_path,
+    )
+    assert len(items) >= 1
+    for item in items:
+        assert item["type"] == "transfer"
+        assert "player" in item
+        assert "direction" in item
+        assert item["direction"] in ("In", "Out", "")
+        assert "club" in item
+        assert "origin_club" in item
+        assert "market_value" in item
+        assert "nationality" in item
+        assert "position" in item
+        assert "age" in item
+        assert "fee" in item
+        assert "source" in item
+        assert "player_href" in item
+        assert item["player_href"] == '' or item["player_href"].startswith("/")
+        assert "parent" in item
+
+    assert any(item["player_href"] for item in items)
+    # club field should be the club name, not a section heading like "Arrivals"
+    direction_words = {'in', 'out', 'arrivals', 'departures'}
+    assert all(item["club"].lower() not in direction_words for item in items)
+
+
+def test_transfers_competition(tmp_path):
+    """Feed a competition transfers page (Croatian 1.HNL) to verify the plus/ layout."""
+    items = run_crawler(
+        "transfers",
+        parents_data={
+            "href": "/1-hnl/transfers/wettbewerb/KR1/plus/?saison_id=2024&s_w=&leihe=1&intern=0&intern=1",
+        },
+        tmp_path=tmp_path,
+    )
+    assert len(items) >= 1
+    for item in items:
+        assert item["type"] == "transfer"
+        assert item["player_href"].startswith("/")
+        assert item["direction"] in ("In", "Out", "")
+
+    assert any(item["age"] for item in items)
+    assert any(item["nationality"] for item in items)
+    assert any(item["position"] for item in items)
+    assert any(item["market_value"] for item in items)
+    assert any(item["club"] for item in items)
+
+
+# ---------------------------------------------------------------------------
 # 8. Error propagation — non-zero exit on failed requests
 # ---------------------------------------------------------------------------
 

tfmkt/cli.py

@@ -13,6 +13,7 @@
     'tournament_editions': 'tfmkt.crawlers.tournament_editions',
     'games': 'tfmkt.crawlers.games',
     'game_lineups': 'tfmkt.crawlers.game_lineups',
+    'transfers': 'tfmkt.crawlers.transfers',
 }
 
 

tfmkt/crawlers/transfers.py

@@ -0,0 +1,160 @@
+import json
+import logging
+import re
+from urllib.parse import urlparse
+
+from crawlee import Request
+
+from tfmkt.common import DEFAULT_BASE_URL, load_parents, create_crawler, check_failures
+
+logger = logging.getLogger(__name__)
+
+
+async def run(parents_arg=None, season=2024, base_url=None):
+    base_url = base_url or DEFAULT_BASE_URL
+
+    crawler, failures = create_crawler()
+
+    parents = load_parents(parents_arg)
+    start_requests = []
+    for item in parents:
+        href = item.get('href')
+        if not href:
+            continue
+        if '/plus/' not in href and '/saison_id/' not in href:
+            if '/transfers/' not in href:
+                href = href.replace('/startseite/', '/transfers/')
+            if '/transfers/' not in href:
+                href = href.rstrip('/') + '/transfers'
+            # Club pages (/verein/) use /saison_id/YEAR; competition pages use /plus/?saison_id=...
+            if '/verein/' in href:
+                href = href.rstrip('/') + f'/saison_id/{season}'
+            else:
+                href = href.rstrip('/') + f'/plus/?saison_id={season}&s_w=&leihe=1&intern=0&intern=1'
+        url = base_url + href
+        start_requests.append(Request.from_url(url=url, label='parse_transfers', user_data={'source': url, 'parent': item}))
+
+    @crawler.router.handler('parse_transfers')
+    async def parse_transfers(context) -> None:
+        sel = context.selector
+
+        tables = sel.xpath("//div[contains(@class,'responsive-table')]//table | //table[contains(@class,'items')]")
+        if not tables:
+            logger.warning('No transfer tables found on %s', context.request.url)
+            return
+
+        for tbl in tables:
+            # string() captures text from nested elements (e.g. linked club names)
+            team_name = tbl.xpath('string(preceding::h2[1])').get()
+            if not team_name or not team_name.strip():
+                team_name = tbl.xpath('string(preceding::h3[1])').get()
+            team_name = team_name.strip() if team_name else ''
+            # On single-club pages the preceding h2 is "Arrivals"/"Departures", not a club name
+            _tn = team_name.lower()
+            if not team_name or any(w in _tn for w in ('arrival', 'departure', 'entrad', 'saíd', 'zugäng', 'abgäng')):
+                team_name = sel.xpath('string(//h1)').get('').strip()
+
+            direction = ''
+            hdr_texts = tbl.xpath('.//thead//tr//th/text()').getall() or []
+            for t in hdr_texts:
+                tt = (t or '').strip().lower()
+                if tt in ('in', 'ins', 'arrivals', 'arrival') or 'in ' in tt:
+                    direction = 'In'
+                    break
+                if tt in ('out', 'outs', 'departures', 'departure') or 'out ' in tt:
+                    direction = 'Out'
+                    break
+            if not direction:
+                # Fallback: some pages label tables with headings rather than header cells
+                dir_candidate = tbl.xpath('string(preceding::h3[1])').get() or tbl.xpath('string(preceding::h2[1])').get()
+                if dir_candidate:
+                    dc = dir_candidate.strip().lower()
+                    if 'in' in dc or 'arrival' in dc or 'ins' in dc:
+                        direction = 'In'
+                    elif 'out' in dc or 'depart' in dc or 'outs' in dc:
+                        direction = 'Out'
+
+            rows = tbl.xpath('./tbody/tr') or tbl.xpath('./tr')
+            for r in rows:
+                if r.xpath('.//th'):
+                    continue
+
+                def cell_text(xpath_expr):
+                    try:
+                        v = r.xpath(xpath_expr).get()
+                        return v.strip() if v else ''
+                    except Exception:
+                        return ''
+
+                player_anchor = r.xpath('.//a[contains(@href, "/spieler/")]')
+                player_href = player_anchor.xpath('@href').get()
+                if not player_href:
+                    continue
+                player = player_anchor.xpath('text()').get('').strip()
+                player_href = player_href.strip()
+                if player_href.startswith('http'):
+                    player_href = urlparse(player_href).path
+
+                age = cell_text('.//td[contains(@class,"alter-transfer-cell")]/text()')
+                if not age:
+                    age = cell_text('./td[@class="zentriert"][1]/text()')
+
+                nationality = r.xpath('.//td[contains(@class,"nat-transfer-cell")]//img/@title').get()
+                if not nationality:
+                    nationality = cell_text('.//td[contains(@class,"nat-transfer-cell")]/text()')
+                if not nationality:
+                    nationality = r.xpath('./td[@class="zentriert"][2]//img/@title').get() or ''
+                if nationality:
+                    nationality = nationality.strip()
+
+                position = cell_text('.//td[contains(@class,"pos-transfer-cell")]/text()')
+                if not position:
+                    position = cell_text('.//td[contains(@class,"kurzpos-transfer-cell")]/text()')
+                if not position:
+                    position = r.xpath('.//table[@class="inline-table"]//tr[2]/td/text()').get() or ''
+
+                market_value = cell_text('.//td[contains(@class,"mw-transfer-cell")]/text()')
+
+                from_club = r.xpath('.//td[contains(@class,"no-border-rechts")]//a/text()').get()
+                if not from_club:
+                    from_club = r.xpath('.//td[contains(@class,"verein-flagge-transfer-cell")]//a/text()').get()
+                if not from_club:
+                    from_club = r.xpath('./td[5]//a/text()').get()
+                if from_club:
+                    from_club = from_club.strip()
+
+                # Fee cells can be multi-line: "Loan fee:<br><i class='normaler-text'>€1.50m</i>"
+                fee_elem = r.xpath('.//td//a[contains(@href,"jumplist")]')
+                if fee_elem:
+                    fee = fee_elem.xpath('string(.)').get()
+                else:
+                    fee = r.xpath('string(.//td[last()])').get()
+                if fee:
+                    fee = re.sub(r"\s+", " ", fee).strip()
+                    fee = re.sub(r":(?=[€$£\d])", ": ", fee)
+                    fee = re.sub(r"(?i)(loan)(?=\s*\d{1,2}/\d{1,2}/\d{2,4})", r"\1 ", fee)
+                else:
+                    fee = ''
+
+                item = {
+                    'type': 'transfer',
+                    'club': team_name or '',
+                    'direction': direction or '',
+                    'player': player or '',
+                    'player_href': player_href or '',
+                    'age': age or '',
+                    'nationality': nationality or '',
+                    'position': position or '',
+                    'market_value': market_value or '',
+                    'origin_club': from_club or '',
+                    'fee': fee or '',
+                }
+                parent = context.request.user_data.get('parent') if hasattr(context.request, 'user_data') else None
+                if parent:
+                    item['parent'] = parent
+                item['source'] = context.request.url
+
+                print(json.dumps(item), flush=True)
+
+    await crawler.run(start_requests)
+    check_failures(failures)