Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 96 additions & 30 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,50 +1,116 @@
```
# Environment variables
.env
.env.local
.env.*

# Python specific
# Python
__pycache__/
*.pyc
*.pyo
*.pyd
.Python
*.so
*.egg-info/
.eggs/

# Dependencies
.venv/
venv/
env/
ENV/
venv/
.venv/
.venv*/
.ENV
.env.bak
.env.dev
.env.development
.env.prod
.env.production
.env.staging
.env.test
.env.testing

# Build artifacts
build/
# Testing
.coverage
coverage/
htmlcov/
.nyc_output/
.pytest_cache/
.tox/
.cover/

# Distribution / packaging
dist/
build/
*.egg
*.egg-info/
.eggs/
pip-log.txt
pip-delete-this-directory.txt

# Logs
*.log

# OS generated files
.DS_Store
Thumbs.db

# Editor files
# IDE
.vscode/
.idea/
*.swp
*.swo

# OS
.DS_Store
Thumbs.db

# Logs
*.log

# Environment
.env
.env.local
*.env.*

# Temp files
*.tmp
*.temp

# Coverage reports
.coverage
htmlcov/
coverage/
# Coverage
coverage.xml
*.gcda
*.gcno

# Testing
.pytest_cache/
# Compiled
*.so
*.dylib
*.dll
*.exe
*.out

# Compressed
*.zip
*.gz
*.tar
*.tgz
*.bz2
*.xz
*.7z
*.rar
*.zst
*.lz4
*.lzh
*.cab
*.arj
*.rpm
*.deb
*.Z
*.lz
*.lzo
*.tar.gz
*.tar.bz2
*.tar.xz
*.tar.zst

# Gradle
.gradle/
build/

# MyPy
.mypy_cache/

# Node
node_modules/
npm-debug.log*
yarn-debug.log*
yarn-error.log*

# Jupyter
.ipynb_checkpoints/

# Docker
.dockerenv
```
3 changes: 2 additions & 1 deletion ai/retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@ def execute(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
)
time.sleep(delay)

# Unreachable in practice, but keeps mypy happy.
# last_exception is guaranteed to be set here because we only reach
# this point if an exception was caught in the loop above.
raise last_exception # type: ignore[misc]

def calculate_delay(self, attempt: int) -> float:
Expand Down
4 changes: 2 additions & 2 deletions services/zc/app/api/v1/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ async def upload_chunk(
form = await request.form()
chunk_index_val = form.get("chunk_index")
else:
chunk_index_val = data.get("chunk_index")
chunk_index_val = data.get("chunk_index") if data else None
Comment on lines 245 to +246

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

If session_id_val is provided in the request headers, the data variable is never initialized. If x-chunk-index is also missing from the headers, accessing data here will raise an UnboundLocalError. To prevent this, we should safely handle the potentially unbound data variable and fall back to reading and parsing the request body if necessary.

Suggested change
else:
chunk_index_val = data.get("chunk_index")
chunk_index_val = data.get("chunk_index") if data else None
else:
try:
data_val = data
except UnboundLocalError:
try:
import json
body = await request.body()
data_val = json.loads(body.decode('utf-8')) if body else None
except Exception:
data_val = None
chunk_index_val = data_val.get("chunk_index") if data_val else None


if chunk_index_val is None:
raise HTTPException(400, "Missing chunk_index")
Expand All @@ -257,7 +257,7 @@ async def upload_chunk(
form = await request.form()
chunk_hash_val = form.get("chunk_hash")
else:
chunk_hash_val = data.get("chunk_hash")
chunk_hash_val = data.get("chunk_hash") if data else None
Comment on lines 259 to +260

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Similar to the chunk_index check, if session_id_val is provided in the headers, data is not defined. If x-chunk-hash is also missing from the headers, accessing data will raise an UnboundLocalError. We should safely handle this and fall back to parsing the request body.

Suggested change
else:
chunk_hash_val = data.get("chunk_hash")
chunk_hash_val = data.get("chunk_hash") if data else None
else:
try:
data_val = data
except UnboundLocalError:
try:
import json
body = await request.body()
data_val = json.loads(body.decode('utf-8')) if body else None
except Exception:
data_val = None
chunk_hash_val = data_val.get("chunk_hash") if data_val else None


if not chunk_hash_val:
raise HTTPException(400, "Missing chunk_hash for integrity verification")
Expand Down
49 changes: 24 additions & 25 deletions services/zc/app/grpc/wire_servicer.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,31 +173,30 @@ async def StreamUploadProgress(
# Check if upload is complete
chunks_received = session.get('chunks_received', 0)

if True:
total_size = session['total_size']
chunk_size = session.get('chunk_size', 1024 * 1024)
total_chunks = session.get('total_chunks', 1)

bytes_transferred = chunks_received * chunk_size
percent_complete = (chunks_received / total_chunks) * 100

# Estimate transfer rate and ETA
elapsed = time.time() - session['started_at']
transfer_rate = (bytes_transferred / elapsed / 1_000_000) if elapsed > 0 else 0
remaining_bytes = total_size - bytes_transferred
eta_seconds = (remaining_bytes / transfer_rate / 1_000_000) if transfer_rate > 0 else 0

yield wire_pb2.UploadProgressStream( # type: ignore[attr-defined]
session_id=session_id,
percent_complete=percent_complete,
bytes_transferred=bytes_transferred,
bytes_total=total_size,
transfer_rate_mbps=transfer_rate,
eta_seconds=int(eta_seconds)
)

if chunks_received >= total_chunks:
break
total_size = session['total_size']
chunk_size = session.get('chunk_size', 1024 * 1024)
total_chunks = session.get('total_chunks', 1)

bytes_transferred = chunks_received * chunk_size
percent_complete = (chunks_received / total_chunks) * 100

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

If total_chunks is 0 (which can happen if total_size is 0), calculating percent_complete will raise a ZeroDivisionError. We should add a guard to handle this case safely.

Suggested change
percent_complete = (chunks_received / total_chunks) * 100
percent_complete = (chunks_received / total_chunks * 100) if total_chunks > 0 else 100.0


# Estimate transfer rate and ETA
elapsed = time.time() - session['started_at']
transfer_rate = (bytes_transferred / elapsed / 1_000_000) if elapsed > 0 else 0
remaining_bytes = total_size - bytes_transferred
eta_seconds = (remaining_bytes / transfer_rate / 1_000_000) if transfer_rate > 0 else 0

yield wire_pb2.UploadProgressStream( # type: ignore[attr-defined]
session_id=session_id,
percent_complete=percent_complete,
bytes_transferred=bytes_transferred,
bytes_total=total_size,
transfer_rate_mbps=transfer_rate,
eta_seconds=int(eta_seconds)
)

if chunks_received >= total_chunks:
break

await asyncio.sleep(0.5)

Expand Down
1 change: 0 additions & 1 deletion services/zc/app/services/upload_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,6 @@ async def _recover_sessions(self) -> None:
"""Recover incomplete upload sessions from Redis."""
CacheKey.build('upload', 'session', '*')
# In production, scan Redis for incomplete sessions
pass

async def init_upload(
self,
Expand Down
1 change: 1 addition & 0 deletions services/zc/scripts/e2e_load_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ def fetch_url(url: str, method: str = "GET", headers: Optional[dict] = None, dat
req.add_header(k, v)

status = 500
response = None
try:
with urllib.request.urlopen(req, timeout=5) as response:
status = response.getcode()
Expand Down
11 changes: 8 additions & 3 deletions services/zc/src/wire/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ def _model(args):

def _read_file(path):
try:
return open(path).read()
with open(path) as f:
return f.read()
except Exception as e:
print(f"[ERROR] Cannot read {path}: {e}", file=sys.stderr)
sys.exit(1)
Expand Down Expand Up @@ -1737,7 +1738,10 @@ def main():
show_stats=args.cache_stats); return
if args.cache:
from wire.zc_cache import cmd_cache_generate
docs = [open(f).read() for f in (args.cache_docs or [])]
docs = []
for f in (args.cache_docs or []):
with open(f) as fh:
docs.append(fh.read())
cmd_cache_generate(args.prompt or "", key, model,
system=args.cache_system or None, docs=docs,
ttl=args.cache_ttl, show_stats=args.cache_stats,
Expand Down Expand Up @@ -2098,7 +2102,8 @@ def main():
file_content=_read_file(args.file) if args.file else None)
print(result)
if args.output:
open(args.output, "w").write(result)
with open(args.output, "w") as fh:
fh.write(result)
return

parser.print_help()
Expand Down
3 changes: 2 additions & 1 deletion services/zc/src/wire/zc_git.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@ def explain_blame(file: str, line_start: int, line_end: int,
cwd: str, api_key: str, model: str) -> str:
blame = _git(f"git log --oneline {file}", cwd)
try:
code = "\n".join(open(f"{cwd}/{file}").readlines()[line_start-1:line_end])
with open(f"{cwd}/{file}") as f:
code = "\n".join(f.readlines()[line_start-1:line_end])

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The lines returned by readlines() already contain trailing newline characters (\n). Joining them with "\n".join(...) introduces extra blank lines between every line of code in the sliced snippet. Using "".join(...) preserves the original formatting.

Suggested change
code = "\n".join(f.readlines()[line_start-1:line_end])
code = "".join(f.readlines()[line_start-1:line_end])

except Exception:
code = "(could not read file)"
return _call(api_key, model,
Expand Down
4 changes: 1 addition & 3 deletions services/zc/tests/test_zc_compliance_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,7 @@ def fake_urlopen(req, timeout=None):
import urllib.error
body = json.dumps({"error": {"type": "permission_error",
"message": "Missing required scopes."}}).encode()
raise urllib.error.HTTPError(req.full_url, 403, "forbidden", {}, None).__class__(
req.full_url, 403, "forbidden", {}, None
) if False else _http_error(req.full_url, 403, body)
return _http_error(req.full_url, 403, body)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

In the mock fake_urlopen, changing raise to return causes the mock to return the HTTPError object instead of raising it. Since urllib.request.urlopen is expected to raise an HTTPError on non-2xx/3xx responses, returning it instead of raising it changes the mock's behavior and may cause the test to fail or behave incorrectly.

Suggested change
return _http_error(req.full_url, 403, body)
raise _http_error(req.full_url, 403, body)


def _http_error(url, code, body):
import io
Expand Down
Loading