GSOC-Week1-Module_A

application/tests/harvester_test/__init__.py

@@ -0,0 +1,3 @@
+"""
+tests for Module A configuration layer (empty for now)
+"""

application/tests/harvester_test/fixtures/duplicate_include_paths.yaml

@@ -0,0 +1,22 @@
+repositories:
+  - id: duplicate-includes
+    type: github
+    enabled: true
+
+    owner: OWASP
+    repo: ASVS
+    branch: master
+
+    paths:
+      include:
+        - "docs/**/*.md"
+        - "docs/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+      overlap_tokens: 100
+
+    polling:
+      mode: incremental
+      interval_minutes: 60

application/tests/harvester_test/fixtures/duplicate_repo_ids.yaml

@@ -0,0 +1,34 @@
+repositories:
+  - id: asvs
+    type: github
+    owner: OWASP
+    repo: ASVS
+
+    paths:
+      include:
+        - "4.0/en/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+
+    polling:
+      mode: incremental
+      interval_minutes: 60
+
+  - id: asvs
+    type: github
+    owner: OWASP
+    repo: CheatSheetSeries
+
+    paths:
+      include:
+        - "cheatsheets/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+
+    polling:
+      mode: incremental
+      interval_minutes: 60

application/tests/harvester_test/fixtures/duplicate_repositories.yaml

@@ -0,0 +1,34 @@
+repositories:
+  - id: asvs-1
+    type: github
+    owner: OWASP
+    repo: ASVS
+
+    paths:
+      include:
+        - "4.0/en/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+
+    polling:
+      mode: incremental
+      interval_minutes: 60
+
+  - id: asvs-2
+    type: github
+    owner: OWASP
+    repo: ASVS
+
+    paths:
+      include:
+        - "4.0/en/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+
+    polling:
+      mode: incremental
+      interval_minutes: 60

application/tests/harvester_test/fixtures/empty_include_paths.yaml

@@ -0,0 +1,16 @@
+repositories:
+  - id: asvs
+    type: github
+    owner: OWASP
+    repo: ASVS
+
+    paths:
+      include: []
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+
+    polling:
+      mode: incremental
+      interval_minutes: 60

application/tests/harvester_test/fixtures/empty_owner.yaml

@@ -0,0 +1,21 @@
+repositories:
+  - id: empty-owner
+    type: github
+    enabled: true
+
+    owner: ""
+    repo: ASVS
+    branch: master
+
+    paths:
+      include:
+        - "docs/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+      overlap_tokens: 100
+
+    polling:
+      mode: incremental
+      interval_minutes: 60

application/tests/harvester_test/fixtures/invalid_chunk_size.yaml

@@ -0,0 +1,18 @@
+repositories:
+  - id: owasp-asvs
+
+    type: github
+    owner: OWASP
+    repo: ASVS
+
+    paths:
+      include:
+        - "4.0/en/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 0
+
+    polling:
+      mode: incremental
+      interval_minutes: 60

application/tests/harvester_test/fixtures/invalid_chunking_strategy.yaml

@@ -0,0 +1,21 @@
+repositories:
+  - id: invalid-strategy
+    type: github
+    enabled: true
+
+    owner: OWASP
+    repo: ASVS
+    branch: master
+
+    paths:
+      include:
+        - "docs/**/*.md"
+
+    chunking:
+      strategy: invalid_strategy
+      max_tokens: 1200
+      overlap_tokens: 100
+
+    polling:
+      mode: incremental
+      interval_minutes: 60

application/tests/harvester_test/fixtures/invalid_missing_id.yaml

@@ -0,0 +1,17 @@
+repositories:
+  - type: github
+
+    owner: OWASP
+    repo: ASVS
+
+    paths:
+      include:
+        - "4.0/en/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+
+    polling:
+      mode: incremental
+      interval_minutes: 60

application/tests/harvester_test/fixtures/invalid_polling_interval.yaml

@@ -0,0 +1,17 @@
+repositories:
+  - id: asvs
+    type: github
+    owner: OWASP
+    repo: ASVS
+
+    paths:
+      include:
+        - "4.0/en/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+
+    polling:
+      mode: incremental
+      interval_minutes: 0

application/tests/harvester_test/fixtures/invalid_polling_mode.yaml

@@ -0,0 +1,21 @@
+repositories:
+  - id: invalid-polling
+    type: github
+    enabled: true
+
+    owner: OWASP
+    repo: ASVS
+    branch: master
+
+    paths:
+      include:
+        - "docs/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+      overlap_tokens: 100
+
+    polling:
+      mode: realtime
+      interval_minutes: 60

application/tests/harvester_test/fixtures/invalid_yaml.yaml

@@ -0,0 +1,4 @@
+repositories:
+  - id: broken
+    paths:
+      include: [unclosed

application/tests/harvester_test/fixtures/valid_repos.yaml

@@ -0,0 +1,16 @@
+repositories:
+  - id: owasp-asvs
+    type: github
+    owner: OWASP
+    repo: ASVS
+    paths:
+      include:
+        - "4.0/en/**/*.md"
+
+    chunking:
+      strategy: markdown
+      max_tokens: 1200
+
+    polling:
+      mode: incremental
+      interval_minutes: 60

application/tests/harvester_test/test_config_loader.py

@@ -0,0 +1,75 @@
+from pathlib import Path
+import pytest
+from application.utils.harvester.config_loader import (
+    ConfigLoaderError,
+    load_repo_config,
+)
+
+FIXTURES_DIR = Path(__file__).parent / "fixtures"
+
+
+def test_load_valid_config():
+    config_path = FIXTURES_DIR / "valid_repos.yaml"
+
+    config = load_repo_config(config_path)
+
+    assert len(config.repositories) == 1
+
+    repo = config.repositories[0]
+
+    assert repo.id == "owasp-asvs"
+    assert repo.owner == "OWASP"
+    assert repo.repo == "ASVS"
+
+
+def test_missing_repository_id():
+    config_path = FIXTURES_DIR / "invalid_missing_id.yaml"
+    with pytest.raises(ConfigLoaderError):
+        load_repo_config(config_path)
+
+
+def test_invalid_chunk_size():
+    config_path = FIXTURES_DIR / "invalid_chunk_size.yaml"
+    with pytest.raises(ConfigLoaderError, match="max_tokens"):
+        load_repo_config(config_path)
+
+
+def test_invalid_yaml_syntax():
+    config_path = FIXTURES_DIR / "invalid_yaml.yaml"
+    with pytest.raises(ConfigLoaderError):
+        load_repo_config(config_path)
+
+
+def test_missing_config_file():
+    with pytest.raises(FileNotFoundError):
+        load_repo_config("does_not_exist.yaml")
+
+
+def test_empty_owner():
+    config_path = FIXTURES_DIR / "empty_owner.yaml"
+
+    with pytest.raises(
+        ConfigLoaderError,
+        match="owner",
+    ):
+        load_repo_config(config_path)
+
+
+def test_invalid_chunking_strategy():
+    config_path = FIXTURES_DIR / "invalid_chunking_strategy.yaml"
+
+    with pytest.raises(
+        ConfigLoaderError,
+        match="strategy",
+    ):
+        load_repo_config(config_path)
+
+
+def test_invalid_polling_mode():
+    config_path = FIXTURES_DIR / "invalid_polling_mode.yaml"
+
+    with pytest.raises(
+        ConfigLoaderError,
+        match="mode",
+    ):
+        load_repo_config(config_path)

application/tests/harvester_test/test_repos_validator.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+import pytest
+from application.utils.harvester.config_loader import (
+    load_repo_config,
+)
+from application.utils.harvester.repos_validator import (
+    RepositoryValidationError,
+    validate_repositories,
+)
+
+FIXTURES_DIR = Path(__file__).parent / "fixtures"
+
+
+def test_duplicate_repository_ids():
+    config_path = FIXTURES_DIR / "duplicate_repo_ids.yaml"
+
+    config = load_repo_config(config_path)
+
+    with pytest.raises(
+        RepositoryValidationError,
+        match="Duplicate repository id",
+    ):
+        validate_repositories(config)
+
+
+def test_duplicate_repositories():
+    config_path = FIXTURES_DIR / "duplicate_repositories.yaml"
+    config = load_repo_config(config_path)
+
+    with pytest.raises(
+        RepositoryValidationError,
+        match="Duplicate repository detected",
+    ):
+        validate_repositories(config)
+
+
+def test_duplicate_include_paths():
+    config_path = FIXTURES_DIR / "duplicate_include_paths.yaml"
+    config = load_repo_config(config_path)
+
+    with pytest.raises(
+        RepositoryValidationError,
+        match="duplicate include paths",
+    ):
+        validate_repositories(config)
+
+
+def test_validate_valid_repositories():
+    config_path = FIXTURES_DIR / "valid_repos.yaml"
+
+    config = load_repo_config(config_path)
+
+    validate_repositories(config)