diff --git a/pyproject.toml b/pyproject.toml index 6004ee9..0ad4f07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "scrapebadger" -version = "0.9.0" +version = "0.10.0" description = "Official Python SDK for ScrapeBadger - Async web scraping APIs for Twitter and more" readme = "README.md" license = { text = "MIT" } diff --git a/src/scrapebadger/__init__.py b/src/scrapebadger/__init__.py index 6c4244e..63c597e 100644 --- a/src/scrapebadger/__init__.py +++ b/src/scrapebadger/__init__.py @@ -47,12 +47,15 @@ async def main(): PostCommentsResponse, PostDetailResponse, PostDuplicatesResponse, + RedditAward, RedditComment, + RedditModeratedSubreddit, RedditPost, RedditRule, RedditSubreddit, RedditTrophy, RedditUser, + RedditUserSubreddit, RedditWikiPage, SearchPostsResponse, SearchUsersResponse, @@ -120,13 +123,16 @@ async def main(): "PostDuplicatesResponse", "RateLimitError", # Reddit core models + "RedditAward", "RedditComment", + "RedditModeratedSubreddit", "RedditPost", "RedditRule", "RedditSubreddit", "RedditTrophy", "RedditUser", "RedditUserProfileResponse", + "RedditUserSubreddit", "RedditWikiPage", # Main client "ScrapeBadger", diff --git a/src/scrapebadger/reddit/__init__.py b/src/scrapebadger/reddit/__init__.py index 2b24805..3759482 100644 --- a/src/scrapebadger/reddit/__init__.py +++ b/src/scrapebadger/reddit/__init__.py @@ -29,13 +29,16 @@ PostCommentsResponse, PostDetailResponse, PostDuplicatesResponse, + RedditAward, RedditComment, + RedditModeratedSubreddit, RedditPagination, RedditPost, RedditRule, RedditSubreddit, RedditTrophy, RedditUser, + RedditUserSubreddit, RedditWikiPage, SearchPostsResponse, SearchUsersResponse, @@ -58,17 +61,20 @@ "PostCommentsResponse", "PostDetailResponse", "PostDuplicatesResponse", + # Reference models + "RedditAward", # Client "RedditClient", # Core models "RedditComment", + "RedditModeratedSubreddit", "RedditPagination", "RedditPost", - # Reference models "RedditRule", "RedditSubreddit", "RedditTrophy", "RedditUser", + "RedditUserSubreddit", "RedditWikiPage", # Response envelopes — search "SearchPostsResponse", diff --git a/src/scrapebadger/reddit/models.py b/src/scrapebadger/reddit/models.py index 39ec465..13d2f5e 100644 --- a/src/scrapebadger/reddit/models.py +++ b/src/scrapebadger/reddit/models.py @@ -5,16 +5,20 @@ Models are organised into: - Core models: RedditPost, RedditComment, RedditSubreddit, RedditUser -- Reference models: RedditRule, RedditWikiPage, RedditTrophy +- Reference models: RedditRule, RedditWikiPage, RedditTrophy, RedditAward +- Nested models: RedditUserSubreddit, RedditModeratedSubreddit - Response envelopes: SearchPostsResponse, SubredditPostsResponse, etc. -Note: As of 0.9.0, Reddit response models are trimmed to the fields available -via the old.reddit.com HTML/RSS source after Reddit deprecated the -unauthenticated .json API. +As of 0.10.0, Reddit response models mirror the full set of content/metadata +fields returned by Reddit's authenticated ``.json`` API. Field names, types, +and defaults match the canonical backend scraper model exactly so that API +responses can be passed through ``model_validate`` without any mapping layer. """ from __future__ import annotations +from typing import Any + from pydantic import BaseModel, ConfigDict, Field # ============================================================================= @@ -33,6 +37,24 @@ class _BaseModel(BaseModel): ) +# ============================================================================= +# Reference / nested models +# ============================================================================= + + +class RedditAward(_BaseModel): + """An award attached to a post/comment (element of ``all_awardings``).""" + + id: str | None = None + name: str = "" + count: int = 0 + coin_price: int = 0 + coin_reward: int = 0 + description: str | None = None + icon_url: str | None = None + award_type: str | None = None + + # ============================================================================= # Core Models # ============================================================================= @@ -43,88 +65,98 @@ class RedditPost(_BaseModel): Field names match the canonical backend model exactly so that API responses can be passed through ``model_validate`` without any mapping layer. - - Attributes: - id: Post ID (base-36, e.g. "abc123"). - fullname: Fully-qualified Reddit name ("t3_abc123"). - title: Post title. - selftext: Body text for self posts. - selftext_html: HTML-rendered body text. - url: Canonical URL of the linked resource or post. - permalink: Relative Reddit URL path. - domain: Domain of the linked URL (or "self."). - author: Username of the post author. - author_fullname: Author's fully-qualified Reddit name. - subreddit: Subreddit name (without r/ prefix). - subreddit_id: Subreddit fullname (e.g. "t5_2qh0y"). - subreddit_name_prefixed: Subreddit name with prefix ("r/python"). - subreddit_type: Subreddit visibility ("public", "restricted", "private"). - score: Net upvotes. - num_comments: Number of top-level comments. - num_crossposts: Number of times crossposted. - created_utc: UTC epoch timestamp of post creation. - created_at: ISO 8601 UTC string of post creation (human-readable). - is_self: Whether this is a self (text) post. - is_gallery: Whether the post is an image gallery. - is_nsfw: Whether the post is marked NSFW. - is_spoiler: Whether the post is marked as a spoiler. - is_stickied: Whether the post is stickied in its subreddit. - is_original_content: Whether the post is marked as OC. - link_flair_text: Post flair display text. - gilded: Number of times gilded. """ + # Identity id: str fullname: str = "" title: str + author: str = "" + author_fullname: str | None = None + # Body selftext: str = "" selftext_html: str | None = None + # Link url: str = "" permalink: str = "" domain: str = "" - author: str = "" - author_fullname: str | None = None + # Subreddit subreddit: str = "" subreddit_id: str | None = None subreddit_name_prefixed: str | None = None subreddit_type: str | None = None + subreddit_subscribers: int = 0 + # Score / engagement score: int = 0 + ups: int = 0 + downs: int = 0 + upvote_ratio: float = 0.0 num_comments: int = 0 num_crossposts: int = 0 + total_awards_received: int = 0 + view_count: int | None = None + gilded: int = 0 + # Timestamps created_utc: float = 0 created_at: str | None = None + edited: bool | float = False + # Content flags / state is_self: bool = False + is_video: bool = False is_gallery: bool = False + is_meta: bool = False + is_original_content: bool = False + is_robot_indexable: bool = True + is_crosspostable: bool = False + is_reddit_media_domain: bool = False + media_only: bool = False is_nsfw: bool = False is_spoiler: bool = False is_stickied: bool = False - is_original_content: bool = False + locked: bool = False + archived: bool = False + pinned: bool = False + quarantine: bool = False + hide_score: bool = False + contest_mode: bool = False + allow_live_comments: bool = False + send_replies: bool = False + distinguished: str | None = None + removed_by_category: str | None = None + suggested_sort: str | None = None + discussion_type: str | None = None + top_awarded_type: str | None = None + # Post flair link_flair_text: str | None = None - gilded: int = 0 + link_flair_type: str | None = None + link_flair_css_class: str | None = None + link_flair_template_id: str | None = None + link_flair_background_color: str | None = None + link_flair_text_color: str | None = None + # Author flair / meta + author_flair_text: str | None = None + author_flair_type: str | None = None + author_flair_css_class: str | None = None + author_flair_template_id: str | None = None + author_flair_background_color: str | None = None + author_flair_text_color: str | None = None + author_premium: bool = False + author_patreon_flair: bool = False + # Media + thumbnail: str | None = None + thumbnail_width: int | None = None + thumbnail_height: int | None = None + media: dict[str, Any] | None = None + media_embed: dict[str, Any] | None = None + secure_media: dict[str, Any] | None = None + secure_media_embed: dict[str, Any] | None = None + # Awards + all_awardings: list[RedditAward] = Field(default_factory=list) + gildings: dict[str, Any] = Field(default_factory=dict) class RedditComment(_BaseModel): - """A Reddit comment. - - Attributes: - id: Comment ID (base-36 string). - fullname: Fully-qualified Reddit name ("t1_"). - body: Comment text body (markdown). - body_html: HTML-rendered comment body. - author: Username of the comment author. - author_fullname: Author's fully-qualified Reddit name. - subreddit: Subreddit the comment belongs to. - subreddit_id: Subreddit fullname (e.g. "t5_2qh0y"). - subreddit_name_prefixed: Subreddit name with prefix ("r/python"). - post_id: ID of the parent post. - permalink: Relative Reddit URL path to the comment. - score: Net upvotes. - depth: Nesting depth in the comment tree (0 = top-level). - created_utc: UTC epoch timestamp of comment creation. - created_at: ISO 8601 UTC string of comment creation. - is_stickied: Whether the comment is stickied. - replies: Nested replies (populated for threaded responses). - """ + """A Reddit comment.""" id: str fullname: str = "" @@ -135,32 +167,49 @@ class RedditComment(_BaseModel): subreddit: str = "" subreddit_id: str | None = None subreddit_name_prefixed: str | None = None + subreddit_type: str | None = None post_id: str | None = None + parent_id: str | None = None permalink: str = "" + # Score / engagement score: int = 0 + ups: int = 0 + downs: int = 0 + controversiality: int = 0 + total_awards_received: int = 0 + gilded: int = 0 + score_hidden: bool = False depth: int = 0 + # Timestamps created_utc: float = 0 created_at: str | None = None + edited: bool | float = False + # Flags / state + is_submitter: bool = False is_stickied: bool = False + locked: bool = False + archived: bool = False + collapsed: bool = False + collapsed_reason: str | None = None + send_replies: bool = False + distinguished: str | None = None + comment_type: str | None = None + # Author flair / meta + author_flair_text: str | None = None + author_flair_type: str | None = None + author_flair_css_class: str | None = None + author_flair_template_id: str | None = None + author_flair_background_color: str | None = None + author_flair_text_color: str | None = None + author_premium: bool = False + # Awards + all_awardings: list[RedditAward] = Field(default_factory=list) + gildings: dict[str, Any] = Field(default_factory=dict) replies: list[RedditComment] = Field(default_factory=list) class RedditSubreddit(_BaseModel): - """A Reddit subreddit (community). - - Attributes: - id: Subreddit ID (base-36). - fullname: Fully-qualified Reddit name ("t5_"). - name: Subreddit name (without r/ prefix). - display_name_prefixed: Name with prefix ("r/python"). - title: Subreddit headline shown in the header. - description: Full sidebar description (markdown). - public_description: Short public description shown in listings. - url: Relative URL path (e.g. "/r/python/"). - created_utc: UTC epoch timestamp of subreddit creation. - created_at: ISO 8601 UTC string of subreddit creation. - is_nsfw: Whether the subreddit is NSFW. - """ + """A Reddit subreddit (community).""" id: str fullname: str = "" @@ -168,35 +217,101 @@ class RedditSubreddit(_BaseModel): display_name_prefixed: str | None = None title: str = "" description: str = "" + description_html: str | None = None public_description: str = "" + public_description_html: str | None = None url: str = "" + subscribers: int = 0 created_utc: float = 0 created_at: str | None = None + # Type / flags + subreddit_type: str | None = None + lang: str | None = None is_nsfw: bool = False + quarantine: bool = False + wiki_enabled: bool = False + over18: bool = False + # Submission settings + submission_type: str | None = None + submit_text: str = "" + submit_text_html: str | None = None + submit_text_label: str | None = None + submit_link_label: str | None = None + allow_images: bool = False + allow_videos: bool = False + allow_galleries: bool = False + allow_polls: bool = False + spoilers_enabled: bool = False + original_content_tag_enabled: bool = False + all_original_content: bool = False + restrict_posting: bool = False + restrict_commenting: bool = False + free_form_reports: bool = True + show_media: bool = False + accept_followers: bool = False + link_flair_enabled: bool = False + link_flair_position: str | None = None + comment_score_hide_mins: int = 0 + suggested_comment_sort: str | None = None + advertiser_category: str | None = None + # Branding + community_icon: str | None = None + icon_img: str | None = None + banner_img: str | None = None + banner_background_image: str | None = None + header_img: str | None = None + header_title: str | None = None + primary_color: str | None = None + key_color: str | None = None + banner_background_color: str | None = None + + +class RedditUserSubreddit(_BaseModel): + """The profile subreddit (u/) embedded in a user's ``about`` payload.""" + + display_name: str = "" + display_name_prefixed: str | None = None + title: str = "" + public_description: str = "" + subscribers: int = 0 + url: str = "" + subreddit_type: str | None = None + over_18: bool = False + icon_img: str | None = None + banner_img: str | None = None + community_icon: str | None = None + primary_color: str | None = None class RedditUser(_BaseModel): - """A Reddit user account. - - Attributes: - name: Username. - display_name_prefixed: Username with prefix ("u/redditor42"). - link_karma: Post karma. - comment_karma: Comment karma. - total_karma: Total karma across all categories. - created_utc: UTC epoch timestamp of account creation. - created_at: ISO 8601 UTC string of account creation. - is_gold: Whether the user has Reddit Premium. - """ + """A Reddit user account.""" + id: str | None = None name: str display_name_prefixed: str | None = None + # Karma link_karma: int = 0 comment_karma: int = 0 + awardee_karma: int = 0 + awarder_karma: int = 0 total_karma: int = 0 + # Timestamps created_utc: float = 0 created_at: str | None = None + # Flags is_gold: bool = False + is_employee: bool = False + is_mod: bool = False + is_friend: bool = False + verified: bool = False + has_verified_email: bool = False + hide_from_robots: bool = False + accept_followers: bool = False + # Avatar + icon_img: str | None = None + snoovatar_img: str | None = None + # Profile subreddit + subreddit: RedditUserSubreddit | None = None # ============================================================================= @@ -205,51 +320,65 @@ class RedditUser(_BaseModel): class RedditRule(_BaseModel): - """A subreddit posting rule. - - Attributes: - priority: Display order (lower = higher priority). - short_name: Short rule name shown in the sidebar. - description: Full rule description (may include markdown). - """ + """A subreddit posting rule.""" priority: int = 0 short_name: str = "" description: str = "" + description_html: str | None = None + violation_reason: str | None = None + created_utc: float = 0 + kind: str | None = None class RedditWikiPage(_BaseModel): - """A subreddit wiki page. - - Attributes: - title: Page title. - content_md: Wiki content as markdown. - content_html: Wiki content as HTML. - revision_by: Username of the last editor. - revision_date: UTC epoch timestamp of the last revision. - """ + """A subreddit wiki page.""" title: str = "" content_md: str = "" content_html: str | None = None revision_by: str | None = None revision_date: float | None = None + revision_id: str | None = None + may_revise: bool = False + reason: str | None = None class RedditTrophy(_BaseModel): - """A Reddit user trophy/award. - - Attributes: - name: Trophy name. - description: Trophy description. - icon_url: Trophy icon image URL. - url: URL associated with the trophy (may be None). - """ + """A Reddit user trophy/award.""" + id: str | None = None + award_id: str | None = None name: str description: str | None = None icon_url: str | None = None + icon_40: str | None = None + icon_70: str | None = None url: str | None = None + granted_at: float | None = None + + +class RedditModeratedSubreddit(_BaseModel): + """A subreddit a user moderates (from ``/user/{u}/moderated_subreddits``).""" + + fullname: str | None = None + sr: str | None = None + name: str = "" + display_name_prefixed: str | None = None + sr_display_name_prefixed: str | None = None + title: str = "" + url: str = "" + subscribers: int = 0 + subreddit_type: str | None = None + over_18: bool = False + community_icon: str | None = None + icon_img: str | None = None + banner_img: str | None = None + primary_color: str | None = None + key_color: str | None = None + created_utc: float = 0 + created_at: str | None = None + mod_permissions: list[str] = Field(default_factory=list) # ============================================================================= @@ -411,7 +540,7 @@ class UserModeratedResponse(_BaseModel): subreddits: List of subreddits the user moderates. """ - subreddits: list[RedditSubreddit] = Field(default_factory=list) + subreddits: list[RedditModeratedSubreddit] = Field(default_factory=list) class UserTrophiesResponse(_BaseModel): @@ -454,10 +583,12 @@ class SubredditRulesResponse(_BaseModel): Attributes: rules: List of subreddit rules. subreddit: Subreddit name. + site_rules: Reddit site-wide rules that also apply. """ rules: list[RedditRule] = Field(default_factory=list) subreddit: str = "" + site_rules: list[str] = Field(default_factory=list) class SubredditWikiPagesResponse(_BaseModel): diff --git a/tests/test_reddit.py b/tests/test_reddit.py index 1fe425c..1952d88 100644 --- a/tests/test_reddit.py +++ b/tests/test_reddit.py @@ -342,51 +342,32 @@ def test_reddit_post_extra_fields_ignored(self) -> None: post = RedditPost.model_validate(data) assert post.id == "abc123" - def test_reddit_post_removed_fields_absent(self) -> None: - """Fields removed in 0.9.0 must not be present on the model.""" + def test_reddit_post_excluded_fields_absent(self) -> None: + """Per-viewer-session and mod-only fields are intentionally NOT exposed. + + Since 0.10.0 the model carries the full .json *content* field set; the + only deliberate omissions are fields tied to the scraping account's + relationship to the content (viewer-session) or mod-only audit fields. + """ post = RedditPost.model_validate(SAMPLE_POST) for attr in ( - "ups", - "downs", - "upvote_ratio", - "view_count", - "num_duplicates", - "edited", - "edited_at", - "is_video", - "is_locked", - "is_archived", - "is_pinned", - "is_robot_indexable", - "is_meta", - "is_crosspostable", - "send_replies", - "link_flair_background_color", - "link_flair_text_color", - "link_flair_template_id", - "link_flair_type", - "link_flair_css_class", - "distinguished", - "thumbnail", - "thumbnail_width", - "thumbnail_height", - "post_hint", - "preview_images", - "media", - "gallery_data", - "crosspost_parent", - "suggested_sort", - "total_awards", - "awards", - "content_categories", - "removed_by_category", - "treatment_tags", - "subreddit_subscribers", - "author_flair_text", - "author_flair_type", - "author_flair_template_id", + "saved", + "likes", + "clicked", + "visited", + "hidden", + "user_reports", + "mod_reports", + "num_reports", + "report_reasons", + "approved_by", + "approved_at_utc", + "banned_by", + "banned_at_utc", + "removed_by", + "mod_note", ): - assert not hasattr(post, attr), f"RedditPost should not have field: {attr}" + assert not hasattr(post, attr), f"RedditPost should not expose viewer/mod field: {attr}" # -- RedditComment -- @@ -439,29 +420,23 @@ def test_reddit_comment_nested_replies(self) -> None: assert len(comment.replies) == 1 assert comment.replies[0].id == "cmt001" - def test_reddit_comment_removed_fields_absent(self) -> None: - """Fields removed in 0.9.0 must not be present on the model.""" + def test_reddit_comment_excluded_fields_absent(self) -> None: + """Per-viewer-session and mod-only comment fields are intentionally omitted.""" comment = RedditComment.model_validate(SAMPLE_COMMENT) for attr in ( - "ups", - "downs", - "controversiality", - "edited", - "edited_at", - "gilded", - "is_locked", - "is_score_hidden", - "is_submitter", - "parent_id", - "post_title", - "send_replies", - "subreddit_type", - "total_awards", - "distinguished", - "author_flair_text", - "author_flair_type", + "saved", + "likes", + "user_reports", + "mod_reports", + "num_reports", + "report_reasons", + "approved_by", + "banned_by", + "mod_note", ): - assert not hasattr(comment, attr), f"RedditComment should not have field: {attr}" + assert not hasattr(comment, attr), ( + f"RedditComment should not expose viewer/mod field: {attr}" + ) # -- RedditSubreddit -- @@ -493,49 +468,20 @@ def test_reddit_subreddit_is_frozen(self) -> None: with pytest.raises(Exception): # noqa: B017 sub.name = "mutated" # type: ignore[misc] - def test_reddit_subreddit_removed_fields_absent(self) -> None: - """Fields removed in 0.9.0 must not be present on the model.""" + def test_reddit_subreddit_excluded_fields_absent(self) -> None: + """Per-viewer (user_*) subreddit fields are intentionally omitted.""" sub = RedditSubreddit.model_validate(SAMPLE_SUBREDDIT) for attr in ( - "subscribers", - "active_users", - "description_html", - "public_description_html", - "submit_text", - "submit_text_html", - "header_title", - "type", - "submission_type", - "is_quarantined", - "is_advertiser_friendly", - "advertiser_category", - "language", - "icon_url", - "header_url", - "banner_url", - "banner_background_color", - "primary_color", - "key_color", - "wiki_enabled", - "allow_images", - "allow_videos", - "allow_galleries", - "allow_polls", - "allow_discovery", - "spoilers_enabled", - "emojis_enabled", - "free_form_reports", - "accept_followers", - "restrict_posting", - "link_flair_enabled", - "link_flair_position", - "user_flair_enabled", - "user_flair_position", - "comment_score_hide_mins", - "should_archive_posts", - "allowed_media_in_comments", + "user_is_banned", + "user_is_contributor", + "user_is_moderator", + "user_is_subscriber", + "user_is_muted", + "user_has_favorited", + "user_flair_text", + "user_sr_theme_enabled", ): - assert not hasattr(sub, attr), f"RedditSubreddit should not have field: {attr}" + assert not hasattr(sub, attr), f"RedditSubreddit should not expose viewer field: {attr}" # -- RedditUser -- @@ -564,31 +510,20 @@ def test_reddit_user_is_frozen(self) -> None: with pytest.raises(Exception): # noqa: B017 user.name = "mutated" # type: ignore[misc] - def test_reddit_user_removed_fields_absent(self) -> None: - """Fields removed in 0.9.0 must not be present on the model.""" + def test_reddit_user_excluded_fields_absent(self) -> None: + """Per-viewer / preference user fields are intentionally omitted.""" user = RedditUser.model_validate(SAMPLE_USER) for attr in ( - "id", - "fullname", - "description", - "icon_url", - "snoovatar_url", - "banner_url", - "profile_title", - "profile_url", - "awardee_karma", - "awarder_karma", - "has_verified_email", - "verified", - "accepts_followers", + "is_blocked", + "accept_pms", + "accept_chats", "has_subscribed", - "is_employee", - "is_mod", - "is_suspended", - "is_nsfw", "pref_show_snoovatar", + "snoovatar_size", ): - assert not hasattr(user, attr), f"RedditUser should not have field: {attr}" + assert not hasattr(user, attr), ( + f"RedditUser should not expose viewer/pref field: {attr}" + ) # -- RedditRule -- @@ -608,11 +543,11 @@ def test_reddit_rule_is_frozen(self) -> None: with pytest.raises(Exception): # noqa: B017 rule.short_name = "mutated" # type: ignore[misc] - def test_reddit_rule_removed_fields_absent(self) -> None: - """Fields removed in 0.9.0 must not be present on the model.""" + def test_reddit_rule_full_fields_present(self) -> None: + """Since 0.10.0 the rule model carries the full .json field set.""" rule = RedditRule.model_validate(SAMPLE_RULE) - for attr in ("description_html", "kind", "violation_reason"): - assert not hasattr(rule, attr), f"RedditRule should not have field: {attr}" + for attr in ("description_html", "kind", "violation_reason", "created_utc"): + assert hasattr(rule, attr), f"RedditRule should expose field: {attr}" # -- RedditWikiPage -- @@ -1195,5 +1130,7 @@ def test_removed_models_not_importable(self) -> None: import importlib models_mod = importlib.import_module("scrapebadger.reddit.models") - for name in ("RedditPreviewImage", "RedditMedia", "RedditAward", "RedditUserSummary"): + # RedditAward was re-introduced in 0.10.0 (all_awardings coverage); the + # other helper models remain removed. + for name in ("RedditPreviewImage", "RedditMedia", "RedditUserSummary"): assert not hasattr(models_mod, name), f"{name} should have been removed in 0.9.0" diff --git a/uv.lock b/uv.lock index 6b33316..dd79088 100644 --- a/uv.lock +++ b/uv.lock @@ -752,7 +752,7 @@ wheels = [ [[package]] name = "scrapebadger" -version = "0.9.0" +version = "0.10.0" source = { editable = "." } dependencies = [ { name = "httpx" },