diff --git a/README.md b/README.md index c229b73..f862db7 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,19 @@ echo '{"type":"competition","competition_type":"world_cup","href":"/world-cup/st # scrape games for UEFA Euro 2024 (season=2023 on Transfermarkt) echo '{"type":"competition","competition_type":"uefa_euro","href":"/uefa-euro/startseite/pokalwettbewerb/EURO","competition_name":"UEFA Euro"}' \ | python -m tfmkt games --season 2023 > euro_2024_games.json + +# scrape transfers for a competition (paste the path from any Transfermarkt URL) +echo '{"href":"/premier-league/transfers/wettbewerb/GB1/plus/?saison_id=2025&s_w=&leihe=1&intern=0&intern=1"}' \ + | python -m tfmkt transfers > transfers.json + +# scrape transfers for a single club +echo '{"href":"/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}' \ + | python -m tfmkt transfers > transfers.json + +# chain: scrape clubs then their transfers +cat competitions.json | head -1 \ + | python -m tfmkt clubs \ + | python -m tfmkt transfers > transfers.json ``` Alternatively you can also use [`dcaribou/transfermarkt-scraper`](https://hub.docker.com/repository/docker/dcaribou/transfermarkt-scraper) docker image @@ -106,6 +119,7 @@ Items are extracted in JSON format with one JSON object per item, which get prin | `tournament_editions` | Competition | Tournament Edition | Historical editions with year, season, winner, coach | | `games` | Competition | Game | Match result, events, managers. Use `--season` to select the edition (e.g. `--season 2021` for Qatar 2022, `--season 2023` for Euro 2024) | | `game_lineups` | Game | Game Lineups | Starting XI, substitutes, formation | +| `transfers` | Club or Competition | Transfer | Arrivals and departures with fee, market value, and player href | Check out [transfermarkt-datasets](https://github.com/dcaribou/transfermarkt-datasets) to see `transfermarkt-scraper` in action on a real project. diff --git a/samples/transfers.json b/samples/transfers.json new file mode 100644 index 0000000..8e93515 --- /dev/null +++ b/samples/transfers.json @@ -0,0 +1,3 @@ +{"type": "transfer", "club": "Cruzeiro Esporte Clube", "direction": "In", "player": "Gerson", "player_href": "/gerson/profil/spieler/341705", "age": "28", "nationality": "Brazil", "position": "Central Midfield", "market_value": "", "origin_club": "Zenit S-Pb", "fee": "\u20ac27.00m", "parent": {"href": "/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}, "source": "https://www.transfermarkt.co.uk/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"} +{"type": "transfer", "club": "Cruzeiro Esporte Clube", "direction": "In", "player": "Keny Arroyo", "player_href": "/keny-arroyo/profil/spieler/1074584", "age": "19", "nationality": "Ecuador", "position": "Right Winger", "market_value": "", "origin_club": "Besiktas", "fee": "\u20ac8.50m", "parent": {"href": "/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}, "source": "https://www.transfermarkt.co.uk/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"} +{"type": "transfer", "club": "Cruzeiro Esporte Clube", "direction": "In", "player": "Marquinhos", "player_href": "/marquinhos/profil/spieler/668268", "age": "22", "nationality": "Brazil", "position": "Right Winger", "market_value": "", "origin_club": "Arsenal", "fee": "\u20ac3.00m", "parent": {"href": "/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"}, "source": "https://www.transfermarkt.co.uk/ec-cruzeiro-belo-horizonte/transfers/verein/609/saison_id/2025"} diff --git a/tests/test_crawlers.py b/tests/test_crawlers.py index 675ea39..10f1d5a 100644 --- a/tests/test_crawlers.py +++ b/tests/test_crawlers.py @@ -468,6 +468,67 @@ def test_tournament_editions(tmp_path): assert item_2022["season"] == "2021" assert item_2022["winner"] == "Argentina" +# --------------------------------------------------------------------------- +# 12. Transfers (1 request — 1 small club) +# --------------------------------------------------------------------------- + +def test_transfers(tmp_path): + """Feed a single small club (HNK Sibenik) to get its transfer activity.""" + items = run_crawler( + "transfers", + parents_data={ + "type": "club", + "href": "/hnk-sibenik/startseite/verein/223", + }, + tmp_path=tmp_path, + ) + assert len(items) >= 1 + for item in items: + assert item["type"] == "transfer" + assert "player" in item + assert "direction" in item + assert item["direction"] in ("In", "Out", "") + assert "club" in item + assert "origin_club" in item + assert "market_value" in item + assert "nationality" in item + assert "position" in item + assert "age" in item + assert "fee" in item + assert "source" in item + assert "player_href" in item + assert item["player_href"] == '' or item["player_href"].startswith("/") + assert "parent" in item + + assert any(item["player_href"] for item in items) + # club field should be the club name, not a section heading like "Arrivals" + direction_words = {'in', 'out', 'arrivals', 'departures'} + assert all(item["club"].lower() not in direction_words for item in items) + + +def test_transfers_competition(tmp_path): + """Feed a competition transfers page (Croatian 1.HNL) to verify the plus/ layout.""" + items = run_crawler( + "transfers", + parents_data={ + "href": "/1-hnl/transfers/wettbewerb/KR1/plus/?saison_id=2024&s_w=&leihe=1&intern=0&intern=1", + }, + tmp_path=tmp_path, + ) + assert len(items) >= 1 + for item in items: + assert item["type"] == "transfer" + assert item["player_href"].startswith("/") + assert item["direction"] in ("In", "Out", "") + + assert any(item["age"] for item in items) + assert any(item["nationality"] for item in items) + assert any(item["position"] for item in items) + assert any(item["market_value"] for item in items) + assert any(item["club"] for item in items) + + +# --------------------------------------------------------------------------- # 8. Error propagation — non-zero exit on failed requests # --------------------------------------------------------------------------- diff --git a/tfmkt/cli.py b/tfmkt/cli.py index 95f88eb..d14b257 100644 --- a/tfmkt/cli.py +++ b/tfmkt/cli.py @@ -13,6 +13,7 @@ 'tournament_editions': 'tfmkt.crawlers.tournament_editions', 'games': 'tfmkt.crawlers.games', 'game_lineups': 'tfmkt.crawlers.game_lineups', + 'transfers': 'tfmkt.crawlers.transfers', } diff --git a/tfmkt/crawlers/transfers.py b/tfmkt/crawlers/transfers.py new file mode 100644 index 0000000..0833d1d --- /dev/null +++ b/tfmkt/crawlers/transfers.py @@ -0,0 +1,160 @@ +import json +import logging +import re +from urllib.parse import urlparse + +from crawlee import Request + +from tfmkt.common import DEFAULT_BASE_URL, load_parents, create_crawler, check_failures + +logger = logging.getLogger(__name__) + + +async def run(parents_arg=None, season=2024, base_url=None): + base_url = base_url or DEFAULT_BASE_URL + + crawler, failures = create_crawler() + + parents = load_parents(parents_arg) + start_requests = [] + for item in parents: + href = item.get('href') + if not href: + continue + if '/plus/' not in href and '/saison_id/' not in href: + if '/transfers/' not in href: + href = href.replace('/startseite/', '/transfers/') + if '/transfers/' not in href: + href = href.rstrip('/') + '/transfers' + # Club pages (/verein/) use /saison_id/YEAR; competition pages use /plus/?saison_id=... + if '/verein/' in href: + href = href.rstrip('/') + f'/saison_id/{season}' + else: + href = href.rstrip('/') + f'/plus/?saison_id={season}&s_w=&leihe=1&intern=0&intern=1' + url = base_url + href + start_requests.append(Request.from_url(url=url, label='parse_transfers', user_data={'source': url, 'parent': item})) + + @crawler.router.handler('parse_transfers') + async def parse_transfers(context) -> None: + sel = context.selector + + tables = sel.xpath("//div[contains(@class,'responsive-table')]//table | //table[contains(@class,'items')]") + if not tables: + logger.warning('No transfer tables found on %s', context.request.url) + return + + for tbl in tables: + # string() captures text from nested elements (e.g. linked club names) + team_name = tbl.xpath('string(preceding::h2[1])').get() + if not team_name or not team_name.strip(): + team_name = tbl.xpath('string(preceding::h3[1])').get() + team_name = team_name.strip() if team_name else '' + # On single-club pages the preceding h2 is "Arrivals"/"Departures", not a club name + _tn = team_name.lower() + if not team_name or any(w in _tn for w in ('arrival', 'departure', 'entrad', 'saíd', 'zugäng', 'abgäng')): + team_name = sel.xpath('string(//h1)').get('').strip() + + direction = '' + hdr_texts = tbl.xpath('.//thead//tr//th/text()').getall() or [] + for t in hdr_texts: + tt = (t or '').strip().lower() + if tt in ('in', 'ins', 'arrivals', 'arrival') or 'in ' in tt: + direction = 'In' + break + if tt in ('out', 'outs', 'departures', 'departure') or 'out ' in tt: + direction = 'Out' + break + if not direction: + # Fallback: some pages label tables with headings rather than header cells + dir_candidate = tbl.xpath('string(preceding::h3[1])').get() or tbl.xpath('string(preceding::h2[1])').get() + if dir_candidate: + dc = dir_candidate.strip().lower() + if 'in' in dc or 'arrival' in dc or 'ins' in dc: + direction = 'In' + elif 'out' in dc or 'depart' in dc or 'outs' in dc: + direction = 'Out' + + rows = tbl.xpath('./tbody/tr') or tbl.xpath('./tr') + for r in rows: + if r.xpath('.//th'): + continue + + def cell_text(xpath_expr): + try: + v = r.xpath(xpath_expr).get() + return v.strip() if v else '' + except Exception: + return '' + + player_anchor = r.xpath('.//a[contains(@href, "/spieler/")]') + player_href = player_anchor.xpath('@href').get() + if not player_href: + continue + player = player_anchor.xpath('text()').get('').strip() + player_href = player_href.strip() + if player_href.startswith('http'): + player_href = urlparse(player_href).path + + age = cell_text('.//td[contains(@class,"alter-transfer-cell")]/text()') + if not age: + age = cell_text('./td[@class="zentriert"][1]/text()') + + nationality = r.xpath('.//td[contains(@class,"nat-transfer-cell")]//img/@title').get() + if not nationality: + nationality = cell_text('.//td[contains(@class,"nat-transfer-cell")]/text()') + if not nationality: + nationality = r.xpath('./td[@class="zentriert"][2]//img/@title').get() or '' + if nationality: + nationality = nationality.strip() + + position = cell_text('.//td[contains(@class,"pos-transfer-cell")]/text()') + if not position: + position = cell_text('.//td[contains(@class,"kurzpos-transfer-cell")]/text()') + if not position: + position = r.xpath('.//table[@class="inline-table"]//tr[2]/td/text()').get() or '' + + market_value = cell_text('.//td[contains(@class,"mw-transfer-cell")]/text()') + + from_club = r.xpath('.//td[contains(@class,"no-border-rechts")]//a/text()').get() + if not from_club: + from_club = r.xpath('.//td[contains(@class,"verein-flagge-transfer-cell")]//a/text()').get() + if not from_club: + from_club = r.xpath('./td[5]//a/text()').get() + if from_club: + from_club = from_club.strip() + + # Fee cells can be multi-line: "Loan fee:
€1.50m" + fee_elem = r.xpath('.//td//a[contains(@href,"jumplist")]') + if fee_elem: + fee = fee_elem.xpath('string(.)').get() + else: + fee = r.xpath('string(.//td[last()])').get() + if fee: + fee = re.sub(r"\s+", " ", fee).strip() + fee = re.sub(r":(?=[€$£\d])", ": ", fee) + fee = re.sub(r"(?i)(loan)(?=\s*\d{1,2}/\d{1,2}/\d{2,4})", r"\1 ", fee) + else: + fee = '' + + item = { + 'type': 'transfer', + 'club': team_name or '', + 'direction': direction or '', + 'player': player or '', + 'player_href': player_href or '', + 'age': age or '', + 'nationality': nationality or '', + 'position': position or '', + 'market_value': market_value or '', + 'origin_club': from_club or '', + 'fee': fee or '', + } + parent = context.request.user_data.get('parent') if hasattr(context.request, 'user_data') else None + if parent: + item['parent'] = parent + item['source'] = context.request.url + + print(json.dumps(item), flush=True) + + await crawler.run(start_requests) + check_failures(failures)